From 4a8b360876e170dd6ff42139b821272ae1dfe022 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Fri, 14 Jul 2017 11:27:59 -0700 Subject: [PATCH 1/2] Create Swift libSyntax API This patch is an initial implementation of the Swift libSyntax API. It aims to provide all features of the C++ API but exposed to Swift. It currently resides in SwiftExperimental and will likely exist in a molten state for a while. --- cmake/modules/SwiftComponents.cmake | 3 +- lib/AST/LegacyASTTransformer.cpp | 5 +- .../StdlibUnittest/StdlibUnittest.swift.gyb | 21 ++ test/SwiftSyntax/LazyCaching.swift | 48 +++ test/SwiftSyntax/ParseFile.swift | 20 ++ test/SwiftSyntax/SyntaxFactory.swift | 113 +++++++ tools/CMakeLists.txt | 5 + tools/SwiftSyntax/AtomicCache.swift | 63 ++++ tools/SwiftSyntax/CMakeLists.txt | 23 ++ tools/SwiftSyntax/README.md | 4 + tools/SwiftSyntax/RawSyntax.swift | 194 ++++++++++++ tools/SwiftSyntax/SourcePresence.swift | 25 ++ tools/SwiftSyntax/SwiftSyntax.swift | 48 +++ tools/SwiftSyntax/SwiftcInvocation.swift | 188 ++++++++++++ tools/SwiftSyntax/Syntax.swift | 217 ++++++++++++++ tools/SwiftSyntax/SyntaxBuilders.swift.gyb | 83 +++++ tools/SwiftSyntax/SyntaxChildren.swift | 30 ++ tools/SwiftSyntax/SyntaxCollection.swift | 121 ++++++++ tools/SwiftSyntax/SyntaxData.swift | 199 ++++++++++++ tools/SwiftSyntax/SyntaxFactory.swift.gyb | 205 +++++++++++++ tools/SwiftSyntax/SyntaxKind.swift.gyb | 81 +++++ tools/SwiftSyntax/SyntaxNodes.swift.gyb | 138 +++++++++ tools/SwiftSyntax/SyntaxRewriter.swift.gyb | 61 ++++ tools/SwiftSyntax/TokenKind.swift.gyb | 112 +++++++ tools/SwiftSyntax/Trivia.swift | 283 ++++++++++++++++++ utils/gyb_syntax_support/Child.py | 4 +- utils/gyb_syntax_support/DeclNodes.py | 19 +- utils/gyb_syntax_support/ExprNodes.py | 7 +- utils/gyb_syntax_support/Node.py | 3 +- utils/gyb_syntax_support/StmtNodes.py | 4 +- utils/gyb_syntax_support/Token.py | 10 + utils/gyb_syntax_support/__init__.py | 17 +- utils/gyb_syntax_support/kinds.py | 20 ++ 33 files changed, 2365 insertions(+), 9 deletions(-) create mode 100644 test/SwiftSyntax/LazyCaching.swift create mode 100644 test/SwiftSyntax/ParseFile.swift create mode 100644 test/SwiftSyntax/SyntaxFactory.swift create mode 100644 tools/SwiftSyntax/AtomicCache.swift create mode 100644 tools/SwiftSyntax/CMakeLists.txt create mode 100644 tools/SwiftSyntax/README.md create mode 100644 tools/SwiftSyntax/RawSyntax.swift create mode 100644 tools/SwiftSyntax/SourcePresence.swift create mode 100644 tools/SwiftSyntax/SwiftSyntax.swift create mode 100644 tools/SwiftSyntax/SwiftcInvocation.swift create mode 100644 tools/SwiftSyntax/Syntax.swift create mode 100644 tools/SwiftSyntax/SyntaxBuilders.swift.gyb create mode 100644 tools/SwiftSyntax/SyntaxChildren.swift create mode 100644 tools/SwiftSyntax/SyntaxCollection.swift create mode 100644 tools/SwiftSyntax/SyntaxData.swift create mode 100644 tools/SwiftSyntax/SyntaxFactory.swift.gyb create mode 100644 tools/SwiftSyntax/SyntaxKind.swift.gyb create mode 100644 tools/SwiftSyntax/SyntaxNodes.swift.gyb create mode 100644 tools/SwiftSyntax/SyntaxRewriter.swift.gyb create mode 100644 tools/SwiftSyntax/TokenKind.swift.gyb create mode 100644 tools/SwiftSyntax/Trivia.swift diff --git a/cmake/modules/SwiftComponents.cmake b/cmake/modules/SwiftComponents.cmake index a63f289e85ddd..ec56a72df7778 100644 --- a/cmake/modules/SwiftComponents.cmake +++ b/cmake/modules/SwiftComponents.cmake @@ -57,6 +57,7 @@ # * stdlib -- the Swift standard library. # * stdlib-experimental -- the Swift standard library module for experimental # APIs. +# * swift-syntax -- the Swift module for the libSyntax Swift API # * sdk-overlay -- the Swift SDK overlay. # * editor-integration -- scripts for Swift integration in IDEs other than # Xcode; @@ -66,7 +67,7 @@ # * toolchain-dev-tools -- install development tools useful in a shared toolchain # * dev -- headers and libraries required to use Swift compiler as a library. set(_SWIFT_DEFINED_COMPONENTS - "autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;llvm-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers") + "autolink-driver;compiler;clang-builtin-headers;clang-resource-dir-symlink;llvm-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;swift-syntax;sdk-overlay;editor-integration;tools;testsuite-tools;toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers") macro(swift_configure_components) # Set the SWIFT_INSTALL_COMPONENTS variable to the default value if it is not passed in via -D diff --git a/lib/AST/LegacyASTTransformer.cpp b/lib/AST/LegacyASTTransformer.cpp index b457b0b2913bf..751678e043374 100644 --- a/lib/AST/LegacyASTTransformer.cpp +++ b/lib/AST/LegacyASTTransformer.cpp @@ -225,7 +225,10 @@ RC LegacyASTTransformer::visitTopLevelCodeDecl(TopLevelCodeDecl *D, const SyntaxData *Parent, const CursorIndex IndexInParent) { - return visitBraceStmt(D->getBody(), Parent, IndexInParent); + auto body = visitBraceStmt(D->getBody(), Parent, IndexInParent); + return SyntaxData::make(RawSyntax::make(SyntaxKind::TopLevelCodeDecl, + {body->getRaw()}, + SourcePresence::Present)); } RC diff --git a/stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb b/stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb index 5c391f32f8060..e069b285fe2e2 100644 --- a/stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb +++ b/stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb @@ -532,6 +532,27 @@ public func expectFalse(_ actual: Bool, ${TRACE}) { } } +public func expectThrows( + _ expectedError: ErrorType? = nil, _ test: () throws -> Void, ${TRACE}) { + do { + try test() + } catch let error as ErrorType { + if let expectedError = expectedError { + expectEqual(expectedError, error) + } + } catch { + expectationFailure("unexpected error thrown: \"\(error)\"", trace: ${trace}) + } +} + +public func expectDoesNotThrow(_ test: () throws -> Void, ${TRACE}) { + do { + try test() + } catch { + expectationFailure("unexpected error thrown: \"\(error)\"", trace: ${trace}) + } +} + public func expectNil(_ value: T?, ${TRACE}) { if value != nil { expectationFailure( diff --git a/test/SwiftSyntax/LazyCaching.swift b/test/SwiftSyntax/LazyCaching.swift new file mode 100644 index 0000000000000..066cea5859f82 --- /dev/null +++ b/test/SwiftSyntax/LazyCaching.swift @@ -0,0 +1,48 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test +// REQUIRES: OS=macosx + +import StdlibUnittest +import Foundation +import Dispatch + +import SwiftSyntax + +var LazyCaching = TestSuite("LazyCaching") + +LazyCaching.test("Pathological") { + let tuple = SyntaxFactory.makeVoidTupleType() + + DispatchQueue.concurrentPerform(iterations: 100) { _ in + expectEqual(tuple.leftParen, tuple.leftParen) + } +} + +LazyCaching.test("TwoAccesses") { + let tuple = SyntaxFactory.makeVoidTupleType() + + let queue1 = DispatchQueue(label: "queue1") + let queue2 = DispatchQueue(label: "queue2") + + var node1: TokenSyntax? + var node2: TokenSyntax? + + let group = DispatchGroup() + queue1.async(group: group) { + node1 = tuple.leftParen + } + queue2.async(group: group) { + node2 = tuple.leftParen + } + group.wait() + + let final = tuple.leftParen + + expectNotNil(node1) + expectNotNil(node2) + expectEqual(node1, node2) + expectEqual(node1, final) + expectEqual(node2, final) +} + +runAllTests() \ No newline at end of file diff --git a/test/SwiftSyntax/ParseFile.swift b/test/SwiftSyntax/ParseFile.swift new file mode 100644 index 0000000000000..6248496e54a5f --- /dev/null +++ b/test/SwiftSyntax/ParseFile.swift @@ -0,0 +1,20 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test +// REQUIRES: OS=macosx + +import Foundation +import StdlibUnittest +import SwiftSyntax + +var ParseFile = TestSuite("ParseFile") + +ParseFile.test("ParseSingleFile") { + let currentFile = URL(fileURLWithPath: #file) + expectDoesNotThrow({ + let currentFileContents = try String(contentsOf: currentFile) + let parsed = try Syntax.parse(currentFile) + expectEqual("\(parsed)", currentFileContents) + }) +} + +runAllTests() \ No newline at end of file diff --git a/test/SwiftSyntax/SyntaxFactory.swift b/test/SwiftSyntax/SyntaxFactory.swift new file mode 100644 index 0000000000000..1f62184386e06 --- /dev/null +++ b/test/SwiftSyntax/SyntaxFactory.swift @@ -0,0 +1,113 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test +// REQUIRES: OS=macosx + +import Foundation +import StdlibUnittest +import SwiftSyntax + +func cannedStructDecl() -> StructDeclSyntax { + let fooID = SyntaxFactory.makeIdentifier("Foo", trailingTrivia: .spaces(1)) + let structKW = SyntaxFactory.makeStructKeyword(trailingTrivia: .spaces(1)) + let rBrace = SyntaxFactory.makeRightBraceToken(leadingTrivia: .newlines(1)) + return StructDeclSyntax { + $0.useStructKeyword(structKW) + $0.useIdentifier(fooID) + $0.useLeftBrace(SyntaxFactory.makeLeftBraceToken()) + $0.useRightBrace(rBrace) + } +} + +var SyntaxFactoryAPI = TestSuite("SyntaxFactoryAPI") + +SyntaxFactoryAPI.test("Generated") { + + let structDecl = cannedStructDecl() + + expectEqual("\(structDecl)", + """ + struct Foo { + } + """) + + let forType = SyntaxFactory.makeIdentifier("for", + leadingTrivia: .backticks(1), + trailingTrivia: [ + .backticks(1), .spaces(1) + ]) + let newBrace = SyntaxFactory.makeRightBraceToken(leadingTrivia: .newlines(2)) + + let renamed = structDecl.withIdentifier(forType) + .withRightBrace(newBrace) + + expectEqual("\(renamed)", + """ + struct `for` { + + } + """) + + expectNotEqual(structDecl.leftBrace, renamed.leftBrace) + expectEqual(structDecl, structDecl.root) + expectNil(structDecl.parent) + expectNotNil(structDecl.leftBrace.parent) + expectEqual(structDecl.leftBrace.parent, structDecl) + + // Ensure that accessing children via named identifiers is exactly the + // same as accessing them as their underlying data. + expectEqual(structDecl.leftBrace, structDecl.child(at: 7)) + + expectEqual("\(structDecl.rightBrace)", + """ + + } + """) +} + +SyntaxFactoryAPI.test("TokenSyntax") { + let tok = SyntaxFactory.makeStructKeyword() + expectEqual("\(tok)", "struct") + expectTrue(tok.isPresent) + + let preSpacedTok = tok.withLeadingTrivia(.spaces(3)) + expectEqual("\(preSpacedTok)", " struct") + + let postSpacedTok = tok.withTrailingTrivia(.spaces(6)) + expectEqual("\(postSpacedTok)", "struct ") + + let prePostSpacedTok = preSpacedTok.withTrailingTrivia(.spaces(4)) + expectEqual("\(prePostSpacedTok)", " struct ") +} + +SyntaxFactoryAPI.test("FunctionCallSyntaxBuilder") { + let string = SyntaxFactory.makeStringLiteralExpr("Hello, world!") + let printID = SyntaxFactory.makeVariableExpr("print") + let arg = FunctionCallArgumentSyntax { + $0.useExpression(string) + } + let call = FunctionCallExprSyntax { + $0.useCalledExpression(printID) + $0.useLeftParen(SyntaxFactory.makeLeftParenToken()) + $0.addFunctionCallArgument(arg) + $0.useRightParen(SyntaxFactory.makeRightParenToken()) + } + expectEqual("\(call)", "print(\"Hello, world!\")") + + let terminatorArg = FunctionCallArgumentSyntax { + $0.useLabel(SyntaxFactory.makeIdentifier("terminator")) + $0.useColon(SyntaxFactory.makeColonToken(trailingTrivia: .spaces(1))) + $0.useExpression(SyntaxFactory.makeStringLiteralExpr(" ")) + } + let callWithTerminator = call.withArgumentList( + SyntaxFactory.makeFunctionCallArgumentList([ + arg.withTrailingComma( + SyntaxFactory.makeCommaToken(trailingTrivia: .spaces(1))), + terminatorArg + ]) + ) + + expectEqual("\(callWithTerminator)", + "print(\"Hello, world!\", terminator: \" \")") +} + +runAllTests() \ No newline at end of file diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 751fba7c6fe20..d57a7de29e577 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -16,9 +16,14 @@ if(SWIFT_BUILD_SOURCEKIT) add_swift_tool_subdirectory(SourceKit) endif() + if(SWIFT_HOST_VARIANT STREQUAL "macosx") # Only build Darwin-specific tools when deploying to OS X. add_swift_tool_subdirectory(swift-stdlib-tool) + + if(SWIFT_BUILD_STDLIB) + add_subdirectory(SwiftSyntax) + endif() endif() diff --git a/tools/SwiftSyntax/AtomicCache.swift b/tools/SwiftSyntax/AtomicCache.swift new file mode 100644 index 0000000000000..5624ac69edcf6 --- /dev/null +++ b/tools/SwiftSyntax/AtomicCache.swift @@ -0,0 +1,63 @@ +//===----------- AtomicCache.swift - Atomically Initialized Cache ---------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// AtomicCache is a wrapper class around an uninitialized value. +/// It takes a closure that it will use to create the value atomically. The +/// value is guaranteed to be set exactly one time, but the provided closure +/// may be called multiple times by threads racing to initialize the value. +/// Do not rely on the closure being called only one time. +class AtomicCache { + /// The cached pointer that will be filled in the first time `value` is + /// accessed. + private var _cachedValue: AnyObject? + + /// The value inside this cache. If the value has not been initialized when + /// this value is requested, then the closure will be called and its resulting + /// value will be atomically compare-exchanged into the cache. + /// If multiple threads access the value before initialization, they will all + /// end up returning the correct, initialized value. + /// - Parameter create: The closure that will return the fully realized value + /// inside the cache. + func value(_ create: () -> Value) -> Value { + return withUnsafeMutablePointer(to: &_cachedValue) { ptr in + // Perform an atomic load -- if we get a value, then return it. + if let _cached = _stdlib_atomicLoadARCRef(object: ptr) { + return _cached as! Value + } + + // Otherwise, create the value... + let value = create() + + // ...and attempt to initialize the pointer with that value. + if _stdlib_atomicInitializeARCRef(object: ptr, desired: value) { + // If we won the race, just return the value we made. + return value + } + + // Otherwise, perform _another_ load to get the up-to-date value, + // and let the one we just made die. + return _stdlib_atomicLoadARCRef(object: ptr) as! Value + } + } + + /// Unsafely attempts to load the value and cast it to the appropriate + /// type. + /// - note: Only for use in the debugger! + @available(*, deprecated, message: "Only for use in the debugger.") + var unsafeValue: Value? { + return withUnsafeMutablePointer(to: &_cachedValue) { + return _stdlib_atomicLoadARCRef(object: $0) as? Value + } + } +} diff --git a/tools/SwiftSyntax/CMakeLists.txt b/tools/SwiftSyntax/CMakeLists.txt new file mode 100644 index 0000000000000..ce594504f0a47 --- /dev/null +++ b/tools/SwiftSyntax/CMakeLists.txt @@ -0,0 +1,23 @@ +add_swift_library(swiftSwiftSyntax SHARED + # This file should be listed the first. Module name is inferred from the + # filename. + SwiftSyntax.swift + AtomicCache.swift + RawSyntax.swift + SourcePresence.swift + SwiftcInvocation.swift + Syntax.swift + SyntaxData.swift + SyntaxChildren.swift + SyntaxCollection.swift + SyntaxBuilders.swift.gyb + SyntaxFactory.swift.gyb + SyntaxKind.swift.gyb + SyntaxNodes.swift.gyb + SyntaxRewriter.swift.gyb + TokenKind.swift.gyb + Trivia.swift + + SWIFT_MODULE_DEPENDS Foundation + INSTALL_IN_COMPONENT swift-syntax + IS_STDLIB) diff --git a/tools/SwiftSyntax/README.md b/tools/SwiftSyntax/README.md new file mode 100644 index 0000000000000..699545cc65486 --- /dev/null +++ b/tools/SwiftSyntax/README.md @@ -0,0 +1,4 @@ +# SwiftLanguage + +This is an in-progress implementation of a Swift API for the +[libSyntax](https://github.com/apple/swift/tree/master/lib/Syntax) library. diff --git a/tools/SwiftSyntax/RawSyntax.swift b/tools/SwiftSyntax/RawSyntax.swift new file mode 100644 index 0000000000000..2f300b9d26ae0 --- /dev/null +++ b/tools/SwiftSyntax/RawSyntax.swift @@ -0,0 +1,194 @@ +//===------------------ RawSyntax.swift - Raw Syntax nodes ----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents the raw tree structure underlying the syntax tree. These nodes +/// have no notion of identity and only provide structure to the tree. They +/// are immutable and can be freely shared between syntax nodes. +indirect enum RawSyntax: Codable { + /// A tree node with a kind, an array of children, and a source presence. + case node(SyntaxKind, [RawSyntax], SourcePresence) + + /// A token with a token kind, leading trivia, trailing trivia, and a source + /// presence. + case token(TokenKind, Trivia, Trivia, SourcePresence) + + /// The syntax kind of this raw syntax. + var kind: SyntaxKind { + switch self { + case .node(let kind, _, _): return kind + case .token(_, _, _, _): return .token + } + } + + var tokenKind: TokenKind? { + switch self { + case .node(_, _, _): return nil + case .token(let kind, _, _, _): return kind + } + } + + /// The layout of the children of this Raw syntax node. + var layout: [RawSyntax] { + switch self { + case .node(_, let layout, _): return layout + case .token(_, _, _, _): return [] + } + } + + /// The source presence of this node. + var presence: SourcePresence { + switch self { + case .node(_, _, let presence), + .token(_, _, _, let presence): return presence + } + } + + /// Whether this node is present in the original source. + var isPresent: Bool { + return presence == .present + } + + /// Whether this node is missing from the original source. + var isMissing: Bool { + return presence == .missing + } + + /// Keys for serializing RawSyntax nodes. + enum CodingKeys: String, CodingKey { + // Keys for the `node` case + case kind, layout, presence + + // Keys for the `token` case + case tokenKind, leadingTrivia, trailingTrivia + } + + /// Creates a RawSyntax from the provided Foundation Decoder. + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let presence = try container.decode(SourcePresence.self, forKey: .presence) + if let kind = try container.decodeIfPresent(SyntaxKind.self, forKey: .kind) { + let layout = try container.decode([RawSyntax].self, forKey: .layout) + self = .node(kind, layout, presence) + } else { + let kind = try container.decode(TokenKind.self, forKey: .tokenKind) + let leadingTrivia = try container.decode(Trivia.self, forKey: .leadingTrivia) + let trailingTrivia = try container.decode(Trivia.self, forKey: .trailingTrivia) + self = .token(kind, leadingTrivia, trailingTrivia, presence) + } + } + + /// Encodes the RawSyntax to the provided Foundation Encoder. + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case let .node(kind, layout, presence): + try container.encode(kind, forKey: .kind) + try container.encode(layout, forKey: .layout) + try container.encode(presence, forKey: .presence) + case let .token(kind, leadingTrivia, trailingTrivia, presence): + try container.encode(kind, forKey: .tokenKind) + try container.encode(leadingTrivia, forKey: .leadingTrivia) + try container.encode(trailingTrivia, forKey: .trailingTrivia) + try container.encode(presence, forKey: .presence) + } + } + + /// Creates a RawSyntax node that's marked missing in the source with the + /// provided kind and layout. + /// - Parameters: + /// - kind: The syntax kind underlying this node. + /// - layout: The children of this node. + /// - Returns: A new RawSyntax `.node` with the provided kind and layout, with + /// `.missing` source presence. + static func missing(_ kind: SyntaxKind) -> RawSyntax { + return .node(kind, [], .missing) + } + + /// Creates a RawSyntax token that's marked missing in the source with the + /// provided kind and no leading/trailing trivia. + /// - Parameter kind: The token kind. + /// - Returns: A new RawSyntax `.token` with the provided kind, no + /// leading/trailing trivia, and `.missing` source presence. + static func missingToken(_ kind: TokenKind) -> RawSyntax { + return .token(kind, [], [], .missing) + } + + /// Returns a new RawSyntax node with the provided layout instead of the + /// existing layout. + /// - Note: This function does nothing with `.token` nodes --- the same token + /// is returned. + /// - Parameter newLayout: The children of the new node you're creating. + func replacingLayout(_ newLayout: [RawSyntax]) -> RawSyntax { + switch self { + case let .node(kind, _, presence): return .node(kind, newLayout, presence) + case .token(_, _, _, _): return self + } + } + + /// Creates a new RawSyntax with the provided child appended to its layout. + /// - Parameter child: The child to append + /// - Note: This function does nothing with `.token` nodes --- the same token + /// is returned. + /// - Return: A new RawSyntax node with the provided child at the end. + func appending(_ child: RawSyntax) -> RawSyntax { + var newLayout = layout + newLayout.append(child) + return replacingLayout(newLayout) + } + + /// Returns the child at the provided cursor in the layout. + /// - Parameter index: The index of the child you're accessing. + /// - Returns: The child at the provided index. + subscript(_ index: CursorType) -> RawSyntax + where CursorType.RawValue == Int { + return layout[index.rawValue] + } + + /// Replaces the child at the provided index in this node with the provided + /// child. + /// - Parameters: + /// - index: The index of the child to replace. + /// - newChild: The new child that should occupy that index in the node. + func replacingChild(_ index: Int, + with newChild: RawSyntax) -> RawSyntax { + precondition(index < layout.count, "cursor \(index) reached past layout") + var newLayout = layout + newLayout[index] = newChild + return replacingLayout(newLayout) + } +} + +extension RawSyntax: TextOutputStreamable { + /// Prints the RawSyntax node, and all of its children, to the provided + /// stream. This implementation must be source-accurate. + /// - Parameter stream: The stream on which to output this node. + func write(to target: inout Target) + where Target: TextOutputStream { + switch self { + case .node(_, let layout, _): + for child in layout { + child.write(to: &target) + } + case let .token(kind, leadingTrivia, trailingTrivia, presence): + guard case .present = presence else { return } + for piece in leadingTrivia { + piece.write(to: &target) + } + target.write(kind.text) + for piece in trailingTrivia { + piece.write(to: &target) + } + } + } +} diff --git a/tools/SwiftSyntax/SourcePresence.swift b/tools/SwiftSyntax/SourcePresence.swift new file mode 100644 index 0000000000000..e69d0e072ae63 --- /dev/null +++ b/tools/SwiftSyntax/SourcePresence.swift @@ -0,0 +1,25 @@ +//===---------------- SourcePresence.swift - Source Presence --------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// An indicator of whether a Syntax node was found or written in the source. +/// +/// A `missing` node does not mean, necessarily, that the source item is +/// considered "implicit", but rather that it was not found in the source. +public enum SourcePresence: String, Codable { + /// The syntax was authored by a human and found, or was generated. + case present = "Present" + + /// The syntax was expected or optional, but not found in the source. + case missing = "Missing" +} diff --git a/tools/SwiftSyntax/SwiftSyntax.swift b/tools/SwiftSyntax/SwiftSyntax.swift new file mode 100644 index 0000000000000..343f705d74c9c --- /dev/null +++ b/tools/SwiftSyntax/SwiftSyntax.swift @@ -0,0 +1,48 @@ +//===--------------- SwiftLanguage.swift - Swift Syntax Library -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// This file provides main entry point into the Syntax library. +//===----------------------------------------------------------------------===// + +import Foundation + +/// A list of possible errors that could be encountered while parsing a +/// Syntax tree. +public enum ParserError: Error { + case swiftcFailed(Int, String) + case invalidFile +} + +extension Syntax { + /// Parses the Swift file at the provided URL into a full-fidelity `Syntax` + /// tree. + /// - Parameter url: The URL you wish to parse. + /// - Returns: A top-level Syntax node representing the contents of the tree, + /// if the parse was successful. + /// - Throws: `ParseError.couldNotFindSwiftc` if `swiftc` could not be + /// located, `ParseError.invalidFile` if the file is invalid. + /// FIXME: Fill this out with all error cases. + public static func parse(_ url: URL) throws -> SourceFileSyntax { + let swiftcRunner = try SwiftcRunner(sourceFile: url) + let result = try swiftcRunner.invoke() + guard result.wasSuccessful else { + throw ParserError.swiftcFailed(result.exitCode, result.stderr) + } + let decoder = JSONDecoder() + let raw = try decoder.decode([RawSyntax].self, from: result.stdoutData) + let topLevelNodes = raw.map { Syntax.fromRaw($0) } + let eof = topLevelNodes.last! as! TokenSyntax + let decls = Array(topLevelNodes.dropLast()) as! [DeclSyntax] + let declList = SyntaxFactory.makeDeclList(decls) + return SyntaxFactory.makeSourceFile(topLevelDecls: declList, + eofToken: eof) + } +} diff --git a/tools/SwiftSyntax/SwiftcInvocation.swift b/tools/SwiftSyntax/SwiftcInvocation.swift new file mode 100644 index 0000000000000..d9a4fc64970d8 --- /dev/null +++ b/tools/SwiftSyntax/SwiftcInvocation.swift @@ -0,0 +1,188 @@ +//===------- SwiftcInvocation.swift - Utilities for invoking swiftc -------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// This file provides the logic for invoking swiftc to parse Swift files. +//===----------------------------------------------------------------------===// + +import Foundation + +#if os(macOS) +import Darwin +#elseif os(Linux) +import Glibc +#endif + +/// The result of process execution, containing the exit status code, +/// stdout, and stderr +struct ProcessResult { + /// The process exit code. A non-zero exit code usually indicates failure. + let exitCode: Int + + /// The contents of the process's stdout as Data. + let stdoutData: Data + + /// The contents of the process's stderr as Data. + let stderrData: Data + + /// The contents of the process's stdout, assuming the data was UTF-8 encoded. + var stdout: String { + return String(data: stdoutData, encoding: .utf8)! + } + + /// The contents of the process's stderr, assuming the data was UTF-8 encoded. + var stderr: String { + return String(data: stderrData, encoding: .utf8)! + } + + /// Whether or not this process had a non-zero exit code. + var wasSuccessful: Bool { + return exitCode == 0 + } +} + +/// Runs the provided executable with the provided arguments and returns the +/// contents of stdout and stderr as Data. +/// - Parameters: +/// - executable: The full file URL to the executable you're running. +/// - arguments: A list of strings to pass to the process as arguments. +/// - Returns: A ProcessResult containing stdout, stderr, and the exit code. +func run(_ executable: URL, arguments: [String] = []) -> ProcessResult { + let stdoutPipe = Pipe() + var stdoutData = Data() + stdoutPipe.fileHandleForReading.readabilityHandler = { file in + stdoutData.append(file.availableData) + } + + let stderrPipe = Pipe() + var stderrData = Data() + stderrPipe.fileHandleForReading.readabilityHandler = { file in + stderrData.append(file.availableData) + } + + let process = Process() + + process.terminationHandler = { process in + stdoutPipe.fileHandleForReading.readabilityHandler = nil + stderrPipe.fileHandleForReading.readabilityHandler = nil + } + + process.launchPath = executable.path + process.arguments = arguments + process.standardOutput = stdoutPipe + process.standardError = stderrPipe + process.launch() + process.waitUntilExit() + return ProcessResult(exitCode: Int(process.terminationStatus), + stdoutData: stdoutData, + stderrData: stderrData) +} + +/// Finds the dylib or executable which the provided address falls in. +/// - Parameter dsohandle: A pointer to a symbol in the object file you're +/// looking for. If not provided, defaults to the +/// caller's `#dsohandle`, which will give you the +/// object file the caller resides in. +/// - Returns: A File URL pointing to the object where the provided address +/// resides. This may be a dylib, shared object, static library, +/// or executable. If unable to find the appropriate object, returns +/// `nil`. +func findFirstObjectFile(for dsohandle: UnsafeRawPointer = #dsohandle) -> URL? { + var info = dl_info() + if dladdr(dsohandle, &info) == 0 { + return nil + } + let path = String(cString: info.dli_fname) + return URL(fileURLWithPath: path) +} + +enum InvocationError: Error { + case couldNotFindSwiftc + case couldNotFindSDK + case abort(code: Int) +} + +struct SwiftcRunner { + /// Gets the `swiftc` binary packaged alongside this library. + /// - Returns: The path to `swiftc` relative to the path of this library + /// file, or `nil` if it could not be found. + /// - Note: This makes assumptions about your Swift installation directory + /// structure. Importantly, it assumes that the directory tree is + /// shaped like this: + /// ``` + /// install_root/ + /// - bin/ + /// - swiftc + /// - lib/ + /// - swift/ + /// - ${target}/ + /// - libswiftSwiftSyntax.[dylib|so] + /// ``` + static func locateSwiftc() -> URL? { + guard let libraryPath = findFirstObjectFile() else { return nil } + let swiftcURL = libraryPath.deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("bin") + .appendingPathComponent("swiftc") + guard FileManager.default.fileExists(atPath: swiftcURL.path) else { + return nil + } + return swiftcURL + } + +#if os(macOS) + /// The location of the macOS SDK, or `nil` if it could not be found. + static let macOSSDK: String? = { + let url = URL(fileURLWithPath: "/usr/bin/env") + let result = run(url, arguments: ["xcrun", "--show-sdk-path"]) + guard result.wasSuccessful else { return nil } + let toolPath = result.stdout.trimmingCharacters(in: .whitespacesAndNewlines) + if toolPath.isEmpty { return nil } + return toolPath + }() +#endif + + /// Internal static cache of the Swiftc path. + static let _swiftcURL: URL? = SwiftcRunner.locateSwiftc() + + /// The URL where the `swiftc` binary lies. + let swiftcURL: URL + + /// The source file being parsed. + let sourceFile: URL + + /// Creates a SwiftcRunner that will parse and emit the syntax + /// tree for a provided source file. + /// - Parameter sourceFile: The URL to the source file you're trying + /// to parse. + init(sourceFile: URL) throws { + guard let url = SwiftcRunner._swiftcURL else { + throw InvocationError.couldNotFindSwiftc + } + self.swiftcURL = url + self.sourceFile = sourceFile + } + + /// Invokes swiftc with the provided arguments. + func invoke() throws -> ProcessResult { + var arguments = ["-frontend", "-emit-syntax"] + arguments.append(sourceFile.path) +#if os(macOS) + guard let sdk = SwiftcRunner.macOSSDK else { + throw InvocationError.couldNotFindSDK + } + arguments.append("-sdk") + arguments.append(sdk) +#endif + return run(swiftcURL, arguments: arguments) + } +} diff --git a/tools/SwiftSyntax/Syntax.swift b/tools/SwiftSyntax/Syntax.swift new file mode 100644 index 0000000000000..827337e579487 --- /dev/null +++ b/tools/SwiftSyntax/Syntax.swift @@ -0,0 +1,217 @@ +//===-------------------- Syntax.swift - Syntax Protocol ------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A Syntax node represents a tree of nodes with tokens at the leaves. +/// Each node has accessors for its known children, and allows efficient +/// iteration over the children through its `children` property. +public class Syntax: CustomStringConvertible { + /// The root of the tree this node is currently in. + internal let _root: SyntaxData + + /// The data backing this node. + /// - note: This is unowned, because the reference to the root data keeps it + /// alive. This means there is an implicit relationship -- the data + /// property must be a descendent of the root. This relationship must + /// be preserved in all circumstances where Syntax nodes are created. + internal unowned var data: SyntaxData + +#if DEBUG + func validate() { + // This is for subclasses to override to perform structural validation. + } +#endif + + /// Creates a Syntax node from the provided root and data. + internal init(root: SyntaxData, data: SyntaxData) { + self._root = root + self.data = data +#if DEBUG + validate() +#endif + } + + /// Access the raw syntax assuming the node is a Syntax. + var raw: RawSyntax { + return data.raw + } + + /// An iterator over children of this node. + public var children: SyntaxChildren { + return SyntaxChildren(node: self) + } + + /// Whether or not this node it marked as `present`. + public var isPresent: Bool { + return raw.presence == .present + } + + /// Whether or not this node it marked as `missing`. + public var isMissing: Bool { + return raw.presence == .missing + } + + /// Whether or not this node represents an Expression. + public var isExpr: Bool { + return raw.kind.isExpr + } + + /// Whether or not this node represents a Declaration. + public var isDecl: Bool { + return raw.kind.isDecl + } + + /// Whether or not this node represents a Statement. + public var isStmt: Bool { + return raw.kind.isStmt + } + + /// Whether or not this node represents a Type. + public var isType: Bool { + return raw.kind.isType + } + + /// Whether or not this node represents a Pattern. + public var isPattern: Bool { + return raw.kind.isPattern + } + + /// The parent of this syntax node, or `nil` if this node is the root. + public var parent: Syntax? { + guard let parentData = data.parent else { return nil } + return Syntax.make(root: _root, data: parentData) + } + + /// The index of this node in the parent's children. + public var indexInParent: Int { + return data.indexInParent + } + + /// The root of the tree in which this node resides. + public var root: Syntax { + return Syntax.make(root: _root, data: _root) + } + + /// Gets the child at the provided index in this node's children. + /// - Parameter index: The index of the child node you're looking for. + /// - Returns: A Syntax node for the provided child, or `nil` if there + /// is not a child at that index in the node. + public func child(at index: Int) -> Syntax? { + guard raw.layout.indices.contains(index) else { return nil } + if raw.layout[index].isMissing { return nil } + return Syntax.make(root: _root, data: data.cachedChild(at: index)) + } + + /// A source-accurate description of this node. + public var description: String { + var s = "" + self.write(to: &s) + return s + } +} + +extension Syntax: TextOutputStreamable { + /// Prints the raw value of this node to the provided stream. + /// - Parameter stream: The stream to which to print the raw tree. + public func write(to target: inout Target) + where Target: TextOutputStream { + data.raw.write(to: &target) + } +} + +extension Syntax: Equatable { + /// Determines if two nodes are equal to each other. + public static func ==(lhs: Syntax, rhs: Syntax) -> Bool { + return lhs.data === rhs.data + } +} + +/// MARK: - Nodes + +/// A Syntax node representing a single token. +public class TokenSyntax: Syntax { + /// The text of the token as written in the source code. + public var text: String { + return tokenKind.text + } + + public func withKind(_ tokenKind: TokenKind) -> TokenSyntax { + guard case let .token(_, leadingTrivia, trailingTrivia, presence) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + let (root, newData) = data.replacingSelf(.token(tokenKind, leadingTrivia, + trailingTrivia, presence)) + return TokenSyntax(root: root, data: newData) + } + + /// Returns a new TokenSyntax with its leading trivia replaced + /// by the provided trivia. + public func withLeadingTrivia(_ leadingTrivia: Trivia) -> TokenSyntax { + guard case let .token(kind, _, trailingTrivia, presence) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, + trailingTrivia, presence)) + return TokenSyntax(root: root, data: newData) + } + + /// Returns a new TokenSyntax with its trailing trivia replaced + /// by the provided trivia. + public func withTrailingTrivia(_ trailingTrivia: Trivia) -> TokenSyntax { + guard case let .token(kind, leadingTrivia, _, presence) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, + trailingTrivia, presence)) + return TokenSyntax(root: root, data: newData) + } + + /// Returns a new TokenSyntax with its leading trivia removed. + public func withoutLeadingTrivia() -> TokenSyntax { + return withLeadingTrivia([]) + } + + /// Returns a new TokenSyntax with its trailing trivia removed. + public func withoutTrailingTrivia() -> TokenSyntax { + return withTrailingTrivia([]) + } + + /// Returns a new TokenSyntax with all trivia removed. + public func withoutTrivia() -> TokenSyntax { + return withoutLeadingTrivia().withoutTrailingTrivia() + } + + /// The leading trivia (spaces, newlines, etc.) associated with this token. + public var leadingTrivia: Trivia { + guard case .token(_, let leadingTrivia, _, _) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + return leadingTrivia + } + + /// The trailing trivia (spaces, newlines, etc.) associated with this token. + public var trailingTrivia: Trivia { + guard case .token(_, _, let trailingTrivia, _) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + return trailingTrivia + } + + /// The kind of token this node represents. + public var tokenKind: TokenKind { + guard case .token(let kind, _, _, _) = raw else { + fatalError("TokenSyntax must have token as its raw") + } + return kind + } +} diff --git a/tools/SwiftSyntax/SyntaxBuilders.swift.gyb b/tools/SwiftSyntax/SyntaxBuilders.swift.gyb new file mode 100644 index 0000000000000..99c5370d1ae3b --- /dev/null +++ b/tools/SwiftSyntax/SyntaxBuilders.swift.gyb @@ -0,0 +1,83 @@ + +%{ + # -*- mode: Swift -*- + from gyb_syntax_support import * + NODE_MAP = create_node_map() + # Ignore the following admonition; it applies to the resulting .swift file only +}% +//// Automatically Generated From SyntaxBuilders.swift.gyb. +//// Do Not Edit Directly! +//===------------ SyntaxBuilders.swift - Syntax Builder definitions -------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +%{ +""" +A builder is a struct with a mutable layout inside. Clients can +specify as many or as few of the children as they want, and the structure is +guaranted to be structurally (if not syntactically) valid. +""" +}% + +% for node in SYNTAX_NODES: +% if node.is_buildable(): +% Builder = node.name + "Builder" +public struct ${Builder} { + private var layout = [ +% for child in node.children: + ${make_missing_swift_child(child)}, +% end + ] + internal init() {} +% for child in node.children: +% child_node = NODE_MAP.get(child.syntax_kind) +% if child_node and child_node.is_syntax_collection(): +% child_elt = child_node.collection_element_name +% child_elt_type = child_node.collection_element_type + + public mutating func add${child_elt}(_ elt: ${child_elt_type}) { + let idx = ${node.name}.Cursor.${child.swift_name}.rawValue + layout[idx] = layout[idx].appending(elt.raw) + } +% else: + + public mutating func use${child.name}(_ node: ${child.type_name}) { + let idx = ${node.name}.Cursor.${child.swift_name}.rawValue + layout[idx] = node.raw + } +% end +% end + + internal func buildData() -> SyntaxData { + return SyntaxData(raw: .node(.${node.swift_syntax_kind}, + layout, .present)) + } +} + +extension ${node.name} { + /// Creates a `${node.name}` using the provided build function. + /// - Parameter: + /// - build: A closure that wil be invoked in order to initialize + /// the fields of the syntax node. + /// This closure is passed a `${Builder}` which you can use to + /// incrementally build the structure of the node. + /// - Returns: A `${node.name}` with all the fields populated in the builder + /// closure. + public convenience init(_ build: (inout ${Builder}) -> Void) { + var builder = ${Builder}() + build(&builder) + let data = builder.buildData() + self.init(root: data, data: data) + } +} + +% end +% end diff --git a/tools/SwiftSyntax/SyntaxChildren.swift b/tools/SwiftSyntax/SyntaxChildren.swift new file mode 100644 index 0000000000000..52333b6e58000 --- /dev/null +++ b/tools/SwiftSyntax/SyntaxChildren.swift @@ -0,0 +1,30 @@ +//===------------- SyntaxChildren.swift - Syntax Child Iterator -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct SyntaxChildren: Sequence { + public struct Iterator: IteratorProtocol { + var index: Int = 0 + let node: Syntax + + public mutating func next() -> Syntax? { + defer { index += 1 } + return node.child(at: index) + } + } + let node: Syntax + + public func makeIterator() -> SyntaxChildren.Iterator { + return Iterator(index: 0, node: node) + } +} diff --git a/tools/SwiftSyntax/SyntaxCollection.swift b/tools/SwiftSyntax/SyntaxCollection.swift new file mode 100644 index 0000000000000..adb4630f5c7e3 --- /dev/null +++ b/tools/SwiftSyntax/SyntaxCollection.swift @@ -0,0 +1,121 @@ +//===-------------- SyntaxCollection.swift - Syntax Collection ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents a collection of Syntax nodes of a specific type. SyntaxCollection +/// behaves as a regular Swift collection, and has accessors that return new +/// versions of the collection with different children. +public class SyntaxCollection: Syntax { + /// Creates a new SyntaxCollection by replacing the underlying layout with + /// a different set of raw syntax nodes. + /// + /// - Parameter layout: The new list of raw syntax nodes underlying this + /// collection. + /// - Returns: A new SyntaxCollection with the new layout underlying it. + internal func replacingLayout( + _ layout: [RawSyntax]) -> SyntaxCollection { + let newRaw = data.raw.replacingLayout(layout) + let (newRoot, newData) = data.replacingSelf(newRaw) + return SyntaxCollection(root: newRoot, data: newData) + } + + /// Creates a new SyntaxCollection by appending the provided syntax element + /// to the children. + /// + /// - Parameter syntax: The element to append. + /// - Returns: A new SyntaxCollection with that element appended to the end. + public func appending( + _ syntax: SyntaxElement) -> SyntaxCollection { + var newLayout = data.raw.layout + newLayout.append(syntax.raw) + return replacingLayout(newLayout) + } + + /// Creates a new SyntaxCollection by prepending the provided syntax element + /// to the children. + /// + /// - Parameter syntax: The element to prepend. + /// - Returns: A new SyntaxCollection with that element prepended to the + /// beginning. + public func prepending( + _ syntax: SyntaxElement) -> SyntaxCollection { + return inserting(syntax, at: 0) + } + + /// Creates a new SyntaxCollection by inserting the provided syntax element + /// at the provided index in the children. + /// + /// - Parameters: + /// - syntax: The element to insert. + /// - index: The index at which to insert the element in the collection. + /// + /// - Returns: A new SyntaxCollection with that element appended to the end. + public func inserting(_ syntax: SyntaxElement, + at index: Int) -> SyntaxCollection { + var newLayout = data.raw.layout + /// Make sure the index is a valid insertion index (0 to 1 past the end) + precondition((newLayout.startIndex...newLayout.endIndex).contains(index), + "inserting node at invalid index \(index)") + newLayout.insert(syntax.raw, at: index) + return replacingLayout(newLayout) + } + + /// Creates a new SyntaxCollection by removing the syntax element at the + /// provided index. + /// + /// - Parameter index: The index of the element to remove from the collection. + /// - Returns: A new SyntaxCollection with the element at the provided index + /// removed. + public func removing(childAt index: Int) -> SyntaxCollection { + var newLayout = data.raw.layout + newLayout.remove(at: index) + return replacingLayout(newLayout) + } + + /// Creates a new SyntaxCollection by removing the first element. + /// + /// - Returns: A new SyntaxCollection with the first element removed. + public func removingFirst() -> SyntaxCollection { + var newLayout = data.raw.layout + newLayout.removeFirst() + return replacingLayout(newLayout) + } + + /// Creates a new SyntaxCollection by removing the last element. + /// + /// - Returns: A new SyntaxCollection with the last element removed. + public func removingLast() -> SyntaxCollection { + var newLayout = data.raw.layout + newLayout.removeLast() + return replacingLayout(newLayout) + } +} + +/// Conformance for SyntaxCollection to the Collection protocol. +extension SyntaxCollection: Collection { + public var startIndex: Int { + return data.childCaches.startIndex + } + + public var endIndex: Int { + return data.childCaches.endIndex + } + + public func index(after i: Int) -> Int { + return data.childCaches.index(after: i) + } + + public subscript(_ index: Int) -> SyntaxElement { + return child(at: index)! as! SyntaxElement + } +} \ No newline at end of file diff --git a/tools/SwiftSyntax/SyntaxData.swift b/tools/SwiftSyntax/SyntaxData.swift new file mode 100644 index 0000000000000..30b4e9c6559aa --- /dev/null +++ b/tools/SwiftSyntax/SyntaxData.swift @@ -0,0 +1,199 @@ +//===-------------------- SyntaxData.swift - Syntax Data ------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A unique identifier for a node in the tree. +/// Currently, this is an index path from the current node to the root of the +/// tree. It's an implementation detail and shouldn't be +/// exposed to clients. +typealias NodeIdentifier = [Int] + +/// SyntaxData is the underlying storage for each Syntax node. +/// It's modelled as an array that stores and caches a SyntaxData for each raw +/// syntax node in its layout. It is up to the specific Syntax nodes to maintain +/// the correct layout of their SyntaxData backing stores. +/// +/// SyntaxData is an implementation detail, and should not be exposed to clients +/// of libSyntax. +/// +/// The following relationships are preserved: +/// parent.cachedChild(at: indexInParent) === self +/// raw.layout.count == childCaches.count +/// pathToRoot.first === indexInParent +final class SyntaxData: Equatable { + + let raw: RawSyntax + let indexInParent: Int + weak var parent: SyntaxData? + + let childCaches: [AtomicCache] + + /// Creates a SyntaxData with the provided raw syntax, pointing to the + /// provided index, in the provided parent. + /// - Parameters: + /// - raw: The raw syntax underlying this node. + /// - indexInParent: The index in the parent's layout where this node will + /// reside. + /// - parent: The parent of this node, or `nil` if this node is the root. + required init(raw: RawSyntax, indexInParent: Int = 0, + parent: SyntaxData? = nil) { + self.raw = raw + self.indexInParent = indexInParent + self.parent = parent + self.childCaches = raw.layout.map { _ in AtomicCache() } + } + + /// The index path from this node to the root. This can be used to uniquely + /// identify this node in the tree. + lazy private(set) var pathToRoot: NodeIdentifier = { + var path = [Int]() + var node = self + while let parent = node.parent { + path.append(node.indexInParent) + node = parent + } + return path + }() + + /// Returns the child data at the provided index in this data's layout. + /// This child is cached and will be used in subsequent accesses. + /// - Note: This function traps if the index is out of the bounds of the + /// data's layout. + /// + /// - Parameter index: The index to create and cache. + /// - Returns: The child's data at the provided index. + func cachedChild(at index: Int) -> SyntaxData { + return childCaches[index].value { realizeChild(index) } + } + + /// Returns the child data at the provided cursor in this data's layout. + /// This child is cached and will be used in subsequent accesses. + /// - Note: This function traps if the cursor is out of the bounds of the + /// data's layout. + /// + /// - Parameter cursor: The cursor to create and cache. + /// - Returns: The child's data at the provided cursor. + func cachedChild( + at cursor: CursorType) -> SyntaxData + where CursorType.RawValue == Int { + return cachedChild(at: cursor.rawValue) + } + + /// Walks up the provided node's parent tree looking for the receiver. + /// - parameter data: The potential child data. + /// - returns: `true` if the receiver exists in the parent chain of the + /// provided data. + /// - seealso: isDescendent(of:) + func isAncestor(of data: SyntaxData) -> Bool { + return data.isDescendent(of: self) + } + + /// Walks up the receiver's parent tree looking for the provided node. + /// - parameter data: The potential ancestor data. + /// - returns: `true` if the data exists in the parent chain of the receiver. + /// - seealso: isAncestor(of:) + func isDescendent(of data: SyntaxData) -> Bool { + if data == self { return true } + var node = self + while let parent = node.parent { + if parent == data { return true } + node = parent + } + return false + } + + /// Creates a copy of `self` and recursively creates `SyntaxData` nodes up to + /// the root. + /// - parameter newRaw: The new RawSyntax that will back the new `Data` + /// - returns: A tuple of both the new root node and the new data with the raw + /// layout replaced. + func replacingSelf( + _ newRaw: RawSyntax) -> (root: SyntaxData, newValue: SyntaxData) { + // If we have a parent already, then ask our current parent to copy itself + // recursively up to the root. + if let parent = parent { + let (root, newParent) = parent.replacingChild(newRaw, at: indexInParent) + let newMe = newParent.cachedChild(at: indexInParent) + return (root: root, newValue: newMe) + } else { + // Otherwise, we're already the root, so return the new data as both the + // new root and the new data. + let newMe = SyntaxData(raw: newRaw, indexInParent: indexInParent, + parent: nil) + return (root: newMe, newValue: newMe) + } + } + + /// Creates a copy of `self` with the child at the provided index replaced + /// with a new SyntaxData containing the raw syntax provided. + /// + /// - Parameters: + /// - child: The raw syntax for the new child to replace. + /// - index: The index pointing to where in the raw layout to place this + /// child. + /// - Returns: The new root node created by this operation, and the new child + /// syntax data. + /// - SeeAlso: replacingSelf(_:) + func replacingChild(_ child: RawSyntax, + at index: Int) -> (root: SyntaxData, newValue: SyntaxData) { + let newRaw = raw.replacingChild(index, with: child) + return replacingSelf(newRaw) + } + + /// Creates a copy of `self` with the child at the provided cursor replaced + /// with a new SyntaxData containing the raw syntax provided. + /// + /// - Parameters: + /// - child: The raw syntax for the new child to replace. + /// - cursor: A cursor that points to the index of the child you wish to + /// replace + /// - Returns: The new root node created by this operation, and the new child + /// syntax data. + /// - SeeAlso: replacingSelf(_:) + func replacingChild(_ child: RawSyntax, + at cursor: CursorType) -> (root: SyntaxData, newValue: SyntaxData) + where CursorType.RawValue == Int { + return replacingChild(child, at: cursor.rawValue) + } + + /// Creates the child's syntax data for the provided cursor. + /// + /// - Parameter cursor: The cursor pointing into the raw syntax's layout for + /// the child you're creating. + /// - Returns: A new SyntaxData for the specific child you're + /// creating, whose parent is pointing to self. + func realizeChild( + _ cursor: CursorType) -> SyntaxData + where CursorType.RawValue == Int { + return realizeChild(cursor.rawValue) + } + + /// Creates the child's syntax data for the provided index. + /// + /// - Parameter cursor: The cursor pointing into the raw syntax's layout for + /// the child you're creating. + /// - Returns: A new SyntaxData for the specific child you're + /// creating, whose parent is pointing to self. + func realizeChild(_ index: Int) -> SyntaxData { + return SyntaxData(raw: raw.layout[index], + indexInParent: index, + parent: self) + } + + /// Tells whether two SyntaxData nodes have the same identity. + /// This is not structural equality. + /// - Returns: True if both datas are exactly the same. + static func ==(lhs: SyntaxData, rhs: SyntaxData) -> Bool { + return lhs === rhs + } +} diff --git a/tools/SwiftSyntax/SyntaxFactory.swift.gyb b/tools/SwiftSyntax/SyntaxFactory.swift.gyb new file mode 100644 index 0000000000000..7c290f94f1778 --- /dev/null +++ b/tools/SwiftSyntax/SyntaxFactory.swift.gyb @@ -0,0 +1,205 @@ +%{ + from gyb_syntax_support import * + # -*- mode: Swift -*- + # Ignore the following admonition it applies to the resulting .swift file only +}% +//// Automatically Generated From SyntaxFactory.swift.gyb. +//// Do Not Edit Directly! +//===------- SyntaxFactory.swift - Syntax Factory implementations ---------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines the SyntaxFactory, one of the most important client-facing +// types in lib/Syntax and likely to be very commonly used. +// +// Effectively a namespace, SyntaxFactory is never instantiated, but is *the* +// one-stop shop for making new Syntax nodes. Putting all of these into a +// collection of static methods provides a single point of API lookup for +// clients' convenience. +// +//===----------------------------------------------------------------------===// + +public enum SyntaxFactory { + public static func makeToken(_ kind: TokenKind, presence: SourcePresence, + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + let data = SyntaxData(raw: .token(kind, leadingTrivia, + trailingTrivia, presence)) + return TokenSyntax(root: data, data: data) + } + + public static func makeUnknownSyntax(tokens: [TokenSyntax]) -> Syntax { + let data = SyntaxData(raw: .node(.unknown, + tokens.map { $0.data.raw }, + .present)) + return Syntax(root: data, data: data) + } + +/// MARK: Syntax Node Creation APIs + +% for node in SYNTAX_NODES: +% if node.children: +% child_params = [] +% for child in node.children: +% param_type = child.type_name +% if child.is_optional: +% param_type = param_type + "?" +% child_params.append("%s: %s" % (child.swift_name, param_type)) +% child_params = ', '.join(child_params) + public static func make${node.syntax_kind}(${child_params}) -> ${node.name} { + let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, [ +% for child in node.children: +% if child.is_optional: + ${child.swift_name}?.data.raw ?? ${make_missing_swift_child(child)}, +% else: + ${child.swift_name}.data.raw, +% end +% end + ], .present)) + return ${node.name}(root: data, data: data) + } +% elif node.is_syntax_collection(): + public static func make${node.syntax_kind}( + _ elements: [${node.collection_element_type}]) -> ${node.name} { + let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, + elements.map { $0.data.raw }, .present)) + return ${node.name}(root: data, data: data) + } +% end + + public static func makeBlank${node.syntax_kind}() -> ${node.name} { + let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, [ +% for child in node.children: + ${make_missing_swift_child(child)}, +% end + ], .present)) + return ${node.name}(root: data, data: data) + } +% end + +/// MARK: Token Creation APIs + +% for token in SYNTAX_TOKENS: +% if token.is_keyword: + public static func make${token.name}Keyword(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + return makeToken(.${token.swift_kind()}, presence: .present, + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } +% elif token.text: + public static func make${token.name}Token(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + return makeToken(.${token.swift_kind()}, presence: .present, + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } +% else: + public static func make${token.name}(_ text: String, + leadingTrivia: Trivia = [], trailingTrivia: Trivia = []) -> TokenSyntax { + return makeToken(.${token.swift_kind()}(text), presence: .present, + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } +% end +% end + +/// MARK: Convenience APIs + + public static func makeVoidTupleType() -> TupleTypeSyntax { + return makeTupleType(leftParen: makeLeftParenToken(), + elements: makeBlankTupleTypeElementList(), + rightParen: makeRightParenToken()) + } + + public static func makeTupleTypeElement(label: TokenSyntax?, + colon: TokenSyntax?, type: TypeSyntax, + comma: TokenSyntax?) -> TupleTypeElementSyntax { + let annotation = makeTypeAnnotation(attributes: makeBlankAttributeList(), + inOutKeyword: nil, + type: type) + return makeTupleTypeElement(label: label, colon: colon, + typeAnnotation: annotation, + comma: comma) + } + + public static func makeTupleTypeElement(type: TypeSyntax, + comma: TokenSyntax?) -> TupleTypeElementSyntax { + return makeTupleTypeElement(label: nil, colon: nil, + type: type, comma: comma) + } + + public static func makeGenericParameter(type: TypeIdentifierSyntax, + trailingComma: TokenSyntax) -> GenericParameterSyntax { + return makeGenericParameter(typeIdentifier: type, colon: nil, + inheritedType: nil, + trailingComma: trailingComma) + } + + public static func makeTypeIdentifier(_ typeName: String, + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TypeIdentifierSyntax { + let identifier = makeIdentifier(typeName, leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + return makeTypeIdentifier(typeName:identifier, genericArgumentClause: nil, + period: nil, typeIdentifier: nil) + } + + public static func makeAnyTypeIdentifier(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TypeIdentifierSyntax { + return makeTypeIdentifier("Any", leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } + + public static func makeSelfTypeIdentifier(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TypeIdentifierSyntax { + return makeTypeIdentifier("Self", leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } + + public static func makeTypeToken(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + return makeIdentifier("Type", leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } + + public static func makeProtocolToken(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + return makeIdentifier("Protocol", leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } + + public static func makeEqualityOperator(leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> TokenSyntax { + return makeToken(.spacedBinaryOperator("=="), + presence: .present, + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + } + + public static func makeStringLiteralExpr(_ text: String, + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> StringLiteralExprSyntax { + let literal = makeStringLiteral("\"\(text)\"", + leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + return makeStringLiteralExpr(stringLiteral: literal) + } + + public static func makeVariableExpr(_ text: String, + leadingTrivia: Trivia = [], + trailingTrivia: Trivia = []) -> SymbolicReferenceExprSyntax { + let string = makeIdentifier(text, + leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia) + return makeSymbolicReferenceExpr(identifier: string, + genericArgumentClause: nil) + } +} diff --git a/tools/SwiftSyntax/SyntaxKind.swift.gyb b/tools/SwiftSyntax/SyntaxKind.swift.gyb new file mode 100644 index 0000000000000..42519f729590e --- /dev/null +++ b/tools/SwiftSyntax/SyntaxKind.swift.gyb @@ -0,0 +1,81 @@ +%{ + from gyb_syntax_support import * + from gyb_syntax_support.kinds import SYNTAX_BASE_KINDS + grouped_nodes = { kind: [] for kind in SYNTAX_BASE_KINDS } + for node in SYNTAX_NODES: + grouped_nodes[node.base_kind].append(node) + # -*- mode: Swift -*- + # Ignore the following admonition; it applies to the resulting .swift file only +}% +//// Automatically Generated From SyntaxKind.swift.gyb. +//// Do Not Edit Directly! +//===--------------- SyntaxKind.swift - Syntax Kind definitions -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Enumerates the known kinds of Syntax represented in the Syntax tree. +internal enum SyntaxKind: String, Codable { + case token = "Token" + case unknown = "Unknown" +% for node in SYNTAX_NODES: + case ${node.swift_syntax_kind} = "${node.syntax_kind}" +% end + +% for name, nodes in grouped_nodes.items(): +% if name not in ["Syntax", "SyntaxCollection"]: + /// Whether the underlying kind is a sub-kind of ${name}Syntax. + public var is${name}: Bool { + switch self { +% for node in nodes: + case .${node.swift_syntax_kind}: return true +% end + default: return false + } + } +% end +% end + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let kind = try container.decode(String.self) + self = SyntaxKind(rawValue: kind) ?? .unknown + } +} + +extension Syntax { + /// Creates a Syntax node from the provided RawSyntax using the appropriate + /// Syntax type, as specified by its kind. + /// - Parameters: + /// - raw: The raw syntax with which to create this node. + /// - root: The root of this tree, or `nil` if the new node is the root. + static func fromRaw(_ raw: RawSyntax) -> Syntax { + let data = SyntaxData(raw: raw) + return make(root: nil, data: data) + } + + /// Creates a Syntax node from the provided SyntaxData using the appropriate + /// Syntax type, as specified by its kind. + /// - Parameters: + /// - root: The root of this tree, or `nil` if the new node is the root. + /// - data: The data for this new node. + static func make(root: SyntaxData?, data: SyntaxData) -> Syntax { + let root = root ?? data + switch data.raw.kind { + case .token: return TokenSyntax(root: root, data: data) + case .unknown: return Syntax(root: root, data: data) +% for node in SYNTAX_NODES: + case .${node.swift_syntax_kind}: return ${node.name}(root: root, data: data) +% end + } + } +} \ No newline at end of file diff --git a/tools/SwiftSyntax/SyntaxNodes.swift.gyb b/tools/SwiftSyntax/SyntaxNodes.swift.gyb new file mode 100644 index 0000000000000..9bc87f4132de4 --- /dev/null +++ b/tools/SwiftSyntax/SyntaxNodes.swift.gyb @@ -0,0 +1,138 @@ +%{ + # -*- mode: Swift -*- + from gyb_syntax_support import * + NODE_MAP = create_node_map() + # Ignore the following admonition it applies to the resulting .swift file only +}% +//// Automatically Generated From SyntaxNodes.swift.gyb. +//// Do Not Edit Directly! +//===------------ SyntaxNodes.swift - Syntax Node definitions -------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +%{ +""" +Each Syntax node is a subclass of a more generic node. For example, +StructDeclSyntax is a subclass of DeclSyntax and can be used in contexts +where DeclSyntax is expected. + +Each node will have: +- An accessor for each child that will lazily instantiate it. +- A `withX(_ x: XSyntax)` method for each child that will return a new Syntax + node with the existing X child replaced with the passed-in version. This is a + way to incrementally transform nodes in the tree. +- An `addX(_ x: XSyntax)` method for children that are collections. This will + append the provided node to the collection and return a new node with that + collection replaced. +- (in DEBUG mode) a `validate()` method that's called in the initializer. This + only validates that all the children are the right kind/token, and does not + raise an error if, say, a non-optional child is missing. +""" +}% + +% for node in SYNTAX_NODES: +% base_type = node.base_type +% if node.collection_element: +public typealias ${node.name} = SyntaxCollection<${node.collection_element_type}> +% else: +public class ${node.name}: ${base_type} { +% if node.children: + enum Cursor: Int { +% for child in node.children: + case ${child.swift_name} +% end + } +% end + +% if node.requires_validation(): +#if DEBUG + override func validate() { + if isMissing { return } + precondition(raw.layout.count == ${len(node.children)}) +% for child in node.children: +% child_var = '_' + child.swift_name + let ${child_var} = raw[Cursor.${child.swift_syntax_kind}] +% if child.token_choices: +% choices = ["." + choice.swift_kind() for choice in child.token_choices] +% choice_array = "[%s]" % ', '.join(choices) + guard let ${child_var}TokenKind = ${child_var}.tokenKind else { + fatalError("expected token child, got \(${child_var}.kind)") + } + precondition(${choice_array}.contains(${child_var}TokenKind), + "expected one of ${choice_array} for '${child.swift_name}' " + + "in node of kind ${node.swift_syntax_kind}") +% elif child.text_choices: +% choices = ", ".join("\"%s\"" % choice +% for choice in child.text_choices) + guard let ${child_var}TokenKind = ${child_var}.tokenKind else { + fatalError("expected token child, got \(${child_var}.kind)") + } + precondition([${choices}].contains(${child_var}TokenKind.text), + "expected one of '[${', '.join(child.text_choices)}]', got " + + "'\(${child_var}TokenKind.text)'") +% else: + precondition(${child_var}.kind == .${child.swift_syntax_kind}, + "expected child of kind .${child.swift_syntax_kind}, " + + "got \(${child_var}.kind)") +% end +% end + } +#endif +% end +% for child in node.children: + +% if child.is_optional: + public var ${child.swift_name}: ${child.type_name}? { + guard raw[Cursor.${child.swift_name}].isPresent else { return nil } + return ${child.type_name}(root: _root, + data: data.cachedChild(at: Cursor.${child.swift_name})) + } +% else: + public var ${(child.swift_name)}: ${child.type_name} { + return ${child.type_name}(root: _root, + data: data.cachedChild(at: Cursor.${child.swift_name})) + } +% end +% child_node = NODE_MAP.get(child.syntax_kind) +% if child_node and child_node.is_syntax_collection(): +% child_elt = child_node.collection_element_name +% child_elt_type = child_node.collection_element_type + + public func add${child_elt}(_ elt: ${child_elt_type}) -> ${node.name} { + let childRaw = raw[Cursor.${child.swift_name}].appending(elt.raw) + let (root, newData) = data.replacingChild(childRaw, + at: Cursor.${child.swift_name}) + return ${node.name}(root: root, data: newData) + } +% end + + public func with${child.name}( + _ new${child.type_name}: ${child.type_name}?) -> ${node.name} { + let raw = new${child.type_name}?.raw ?? ${make_missing_swift_child(child)} + let (root, newData) = data.replacingChild(raw, + at: Cursor.${child.swift_name}) + return ${node.name}(root: root, data: newData) + } +% end +} +% end +% end + +/// MARK: Convenience methods + +extension StructDeclSyntax { + func withIdentifier(_ name: String) -> StructDeclSyntax { + let newToken = SyntaxFactory.makeIdentifier(name, + leadingTrivia: identifier.leadingTrivia, + trailingTrivia: identifier.trailingTrivia) + return withIdentifier(newToken) + } +} \ No newline at end of file diff --git a/tools/SwiftSyntax/SyntaxRewriter.swift.gyb b/tools/SwiftSyntax/SyntaxRewriter.swift.gyb new file mode 100644 index 0000000000000..74b87f005a743 --- /dev/null +++ b/tools/SwiftSyntax/SyntaxRewriter.swift.gyb @@ -0,0 +1,61 @@ +%{ + from gyb_syntax_support import * + # -*- mode: Swift -*- + # Ignore the following admonition it applies to the resulting .swift file only + def is_visitable(node): + return not node.is_base() and not node.collection_element +}% +//// Automatically Generated From SyntaxFactory.swift.gyb. +//// Do Not Edit Directly! +//===------------ SyntaxRewriter.swift - Syntax Rewriter class ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines the SyntaxRewriter, a class that performs a standard walk +// and tree-rebuilding pattern. +// +// Subclassers of this class can override the walking behavior for any syntax +// node and transform nodes however they like. +// +//===----------------------------------------------------------------------===// + +open class SyntaxRewriter { + public init() {} +% for node in SYNTAX_NODES: +% if is_visitable(node): + open func visit(_ node: ${node.name}) -> ${node.base_type} { +% cast = ('as! ' + node.base_type) if node.base_type != 'Syntax' else '' + return visitChildren(node)${cast} + } + +% end +% end + + open func visit(_ token: TokenSyntax) -> Syntax { + return token + } + public func visit(_ node: Syntax) -> Syntax { + switch node.raw.kind { + case .token: return visit(node as! TokenSyntax) +% for node in SYNTAX_NODES: +% if is_visitable(node): + case .${node.swift_syntax_kind}: return visit(node as! ${node.name}) +% end +% end + default: return visitChildren(node) + } + } + + func visitChildren(_ node: Syntax) -> Syntax { + let newLayout = node.children.map { visit($0).raw } + return Syntax.fromRaw(node.raw.replacingLayout(newLayout)) + } +} diff --git a/tools/SwiftSyntax/TokenKind.swift.gyb b/tools/SwiftSyntax/TokenKind.swift.gyb new file mode 100644 index 0000000000000..39c65ce96a9c1 --- /dev/null +++ b/tools/SwiftSyntax/TokenKind.swift.gyb @@ -0,0 +1,112 @@ +%{ + # -*- mode: Swift -*- + from gyb_syntax_support import * + # Ignore the following admonition it applies to the resulting .swift file only +}% +//// Automatically Generated From TokenKind.swift.gyb. +//// Do Not Edit Directly! +//===----------------- TokenKind.swift - Token Kind Enum ------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Enumerates the kinds of tokens in the Swift language. +public enum TokenKind: Codable { + case unknown + case eof +% for token in SYNTAX_TOKENS: +% kind = token.swift_kind() +% +% # Tokens that don't have a set text have an associated value that +% # contains their text. +% if not token.text: +% kind += '(String)' +% end + case ${kind} +% end + + /// The textual representation of this token kind. + var text: String { + switch self { + case .unknown: return "unknown" + case .eof: return "" +% for token in SYNTAX_TOKENS: +% if token.text: + case .${token.swift_kind()}: return "${token.text}" +% else: + case .${token.swift_kind()}(let text): return text +% end +% end + } + } + + /// Keys for serializing and deserializing token kinds. + enum CodingKeys: String, CodingKey { + case kind, text + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "unknown": self = .unknown + case "eof": self = .eof +% for token in SYNTAX_TOKENS: + case "${token.kind}": +% if token.text: + self = .${token.swift_kind()} +% else: + let text = try container.decode(String.self, forKey: .text) + self = .${token.swift_kind()}(text) +% end +% end + default: fatalError("unknown token kind \(kind)") + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(kind, forKey: .kind) + try container.encode(text, forKey: .text) + } + + var kind: String { + switch self { + case .unknown: return "unknown" + case .eof: return "eof" +% for token in SYNTAX_TOKENS: +% kind = token.swift_kind() +% if not token.text: +% kind += '(_)' +% end + case .${kind}: return "${token.kind}" +% end + } + } +} + +extension TokenKind: Equatable { + public static func ==(lhs: TokenKind, rhs: TokenKind) -> Bool { + switch (lhs, rhs) { + case (.unknown, .unknown): return true + case (.eof, .eof): return true +% for token in SYNTAX_TOKENS: +% kind = token.swift_kind() +% if token.text: + case (.${kind}, .${kind}): return true +% else: + case (.${kind}(let lhsText), .${kind}(let rhsText)): + return lhsText == rhsText +% end +% end + default: return false + } + } +} diff --git a/tools/SwiftSyntax/Trivia.swift b/tools/SwiftSyntax/Trivia.swift new file mode 100644 index 0000000000000..af3ea426c38df --- /dev/null +++ b/tools/SwiftSyntax/Trivia.swift @@ -0,0 +1,283 @@ +//===------------------- Trivia.swift - Source Trivia Enum ----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// A contiguous stretch of a single kind of trivia. The constituent part of +/// a `Trivia` collection. +/// +/// For example, four spaces would be represented by +/// `.spaces(4)` +/// +/// In general, you should deal with the actual Trivia collection instead +/// of individual pieces whenever possible. +public enum TriviaPiece: Codable { + enum CodingKeys: CodingKey { + case kind, value + } + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "Space": + let value = try container.decode(Int.self, forKey: .value) + self = .spaces(value) + case "Tab": + let value = try container.decode(Int.self, forKey: .value) + self = .tabs(value) + case "VerticalTab": + let value = try container.decode(Int.self, forKey: .value) + self = .verticalTabs(value) + case "Formfeed": + let value = try container.decode(Int.self, forKey: .value) + self = .formfeeds(value) + case "Newline": + let value = try container.decode(Int.self, forKey: .value) + self = .newlines(value) + case "Backtick": + let value = try container.decode(Int.self, forKey: .value) + self = .backticks(value) + case "LineComment": + let value = try container.decode(String.self, forKey: .value) + self = .lineComment(value) + case "BlockComment": + let value = try container.decode(String.self, forKey: .value) + self = .blockComment(value) + case "DocLineComment": + let value = try container.decode(String.self, forKey: .value) + self = .docLineComment(value) + case "DocBlockComment": + let value = try container.decode(String.self, forKey: .value) + self = .docLineComment(value) + default: + let context = + DecodingError.Context(codingPath: [CodingKeys.kind], + debugDescription: "invalid TriviaPiece kind \(kind)") + throw DecodingError.valueNotFound(String.self, context) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .blockComment(let comment): + try container.encode("BlockComment", forKey: .kind) + try container.encode(comment, forKey: .value) + case .docBlockComment(let comment): + try container.encode("DocBlockComment", forKey: .kind) + try container.encode(comment, forKey: .value) + case .docLineComment(let comment): + try container.encode("DocLineComment", forKey: .kind) + try container.encode(comment, forKey: .value) + case .lineComment(let comment): + try container.encode("LineComment", forKey: .kind) + try container.encode(comment, forKey: .value) + case .formfeeds(let count): + try container.encode("Formfeed", forKey: .kind) + try container.encode(count, forKey: .value) + case .backticks(let count): + try container.encode("Backtick", forKey: .kind) + try container.encode(count, forKey: .value) + case .newlines(let count): + try container.encode("Newline", forKey: .kind) + try container.encode(count, forKey: .value) + case .spaces(let count): + try container.encode("Space", forKey: .kind) + try container.encode(count, forKey: .value) + case .tabs(let count): + try container.encode("Tab", forKey: .kind) + try container.encode(count, forKey: .value) + case .verticalTabs(let count): + try container.encode("VerticalTab", forKey: .kind) + try container.encode(count, forKey: .value) + + } + } + + /// A space ' ' character. + case spaces(Int) + + /// A tab '\t' character. + case tabs(Int) + + /// A vertical tab '\v' character. + case verticalTabs(Int) + + /// A form-feed '\f' character. + case formfeeds(Int) + + /// A newline '\n' character. + case newlines(Int) + + /// A backtick '`' character, used to escape identifiers. + case backticks(Int) + + /// A developer line comment, starting with '//' + case lineComment(String) + + /// A developer block comment, starting with '/*' and ending with '*/'. + case blockComment(String) + + /// A documentation line comment, starting with '///'. + case docLineComment(String) + + /// A documentation block comment, starting with '/**' and ending with '*/. + case docBlockComment(String) +} + +extension TriviaPiece: TextOutputStreamable { + /// Prints the provided trivia as they would be written in a source file. + /// + /// - Parameter stream: The stream to which to print the trivia. + public func write(to target: inout Target) + where Target: TextOutputStream { + func printRepeated(_ character: String, count: Int) { + for _ in 0.. Trivia { + var copy = pieces + copy.append(piece) + return Trivia(pieces: copy) + } + + /// Return a piece of trivia for some number of space characters in a row. + public static func spaces(_ count: Int) -> Trivia { + return [.spaces(count)] + } + + /// Return a piece of trivia for some number of tab characters in a row. + public static func tabs(_ count: Int) -> Trivia { + return [.tabs(count)] + } + + /// A vertical tab '\v' character. + public static func verticalTabs(_ count: Int) -> Trivia { + return [.verticalTabs(count)] + } + + /// A form-feed '\f' character. + public static func formfeeds(_ count: Int) -> Trivia { + return [.formfeeds(count)] + } + + /// Return a piece of trivia for some number of newline characters + /// in a row. + public static func newlines(_ count: Int) -> Trivia { + return [.newlines(count)] + } + + /// Return a piece of trivia for some number of backtick '`' characters + /// in a row. + public static func backticks(_ count: Int) -> Trivia { + return [.backticks(count)] + } + + /// Return a piece of trivia for a single line of ('//') developer comment. + public static func lineComment(_ text: String) -> Trivia { + return [.lineComment(text)] + } + + /// Return a piece of trivia for a block comment ('/* ... */') + public static func blockComment(_ text: String) -> Trivia { + return [.blockComment(text)] + } + + /// Return a piece of trivia for a single line of ('///') doc comment. + public static func docLineComment(_ text: String) -> Trivia { + return [.docLineComment(text)] + } + + /// Return a piece of trivia for a documentation block comment ('/** ... */') + public static func docBlockComment(_ text: String) -> Trivia { + return [.docBlockComment(text)] + } +} + +/// Conformance for Trivia to the Collection protocol. +extension Trivia: Collection { + public var startIndex: Int { + return pieces.startIndex + } + + public var endIndex: Int { + return pieces.endIndex + } + + public func index(after i: Int) -> Int { + return pieces.index(after: i) + } + + public subscript(_ index: Int) -> TriviaPiece { + return pieces[index] + } +} + + +extension Trivia: ExpressibleByArrayLiteral { + /// Creates Trivia from the provided pieces. + public init(arrayLiteral elements: TriviaPiece...) { + self.pieces = elements + } +} + +/// Concatenates two collections of `Trivia` into one collection. +public func +(lhs: Trivia, rhs: Trivia) -> Trivia { + return Trivia(pieces: lhs.pieces + rhs.pieces) +} diff --git a/utils/gyb_syntax_support/Child.py b/utils/gyb_syntax_support/Child.py index f77d2e780aab5..9a0eb74470451 100644 --- a/utils/gyb_syntax_support/Child.py +++ b/utils/gyb_syntax_support/Child.py @@ -1,6 +1,6 @@ # flake8: noqa I201 from Token import SYNTAX_TOKEN_MAP -from kinds import kind_to_type +from kinds import kind_to_type, lowercase_first_word class Child(object): @@ -11,7 +11,9 @@ class Child(object): def __init__(self, name, kind, is_optional=False, token_choices=None, text_choices=None): self.name = name + self.swift_name = lowercase_first_word(name) self.syntax_kind = kind + self.swift_syntax_kind = lowercase_first_word(self.syntax_kind) self.type_name = kind_to_type(self.syntax_kind) # If the child has "token" anywhere in the kind, it's considered diff --git a/utils/gyb_syntax_support/DeclNodes.py b/utils/gyb_syntax_support/DeclNodes.py index 0cca6f35ac437..bc0202aaa9052 100644 --- a/utils/gyb_syntax_support/DeclNodes.py +++ b/utils/gyb_syntax_support/DeclNodes.py @@ -115,7 +115,7 @@ Child('AccessLevelModifier', kind='AccessLevelModifier', is_optional=True), Child('StructKeyword', kind='StructToken'), - Child('Name', kind='IdentifierToken'), + Child('Identifier', kind='IdentifierToken'), Child('GenericParameterClause', kind='GenericParameterClause', is_optional=True), Child('InheritanceClause', kind='TypeInheritanceClause', @@ -126,6 +126,23 @@ Child('Members', kind='StructMembers'), Child('RightBrace', kind='RightBraceToken'), ]), + + # decl-list = decl decl-list? + Node('DeclList', kind='SyntaxCollection', + element='Decl'), + + # source-file = decl-list eof + Node('SourceFile', kind='Syntax', + children=[ + Child('TopLevelDecls', kind='DeclList'), + Child('EOFToken', kind='EOFToken') + ]), + + # top-level-code-decl = stmt-list + Node('TopLevelCodeDecl', kind='Decl', + children=[ + Child('Body', kind='StmtList') + ]), # parameter -> # external-parameter-name? local-parameter-name ':' diff --git a/utils/gyb_syntax_support/ExprNodes.py b/utils/gyb_syntax_support/ExprNodes.py index caa036619e875..07a9d5aff9fe9 100644 --- a/utils/gyb_syntax_support/ExprNodes.py +++ b/utils/gyb_syntax_support/ExprNodes.py @@ -65,7 +65,7 @@ # !true Node('PrefixOperatorExpr', kind='Expr', children=[ - Child('Operator', kind='PrefixOperatorToken', + Child('OperatorToken', kind='PrefixOperatorToken', is_optional=True), Child('PostfixExpression', kind='Expr'), ]), @@ -111,4 +111,9 @@ is_optional=True), Child('Digits', kind='IntegerLiteralToken'), ]), + + Node('StringLiteralExpr', kind='Expr', + children=[ + Child("StringLiteral", kind='StringLiteralToken') + ]) ] diff --git a/utils/gyb_syntax_support/Node.py b/utils/gyb_syntax_support/Node.py index 1d2a6688c43fc..52267f2abdb84 100644 --- a/utils/gyb_syntax_support/Node.py +++ b/utils/gyb_syntax_support/Node.py @@ -1,6 +1,6 @@ from __future__ import print_function import sys # noqa: I201 -from kinds import SYNTAX_BASE_KINDS, kind_to_type +from kinds import SYNTAX_BASE_KINDS, kind_to_type, lowercase_first_word def error(msg): @@ -19,6 +19,7 @@ class Node(object): def __init__(self, name, kind=None, children=None, element=None, element_name=None): self.syntax_kind = name + self.swift_syntax_kind = lowercase_first_word(name) self.name = kind_to_type(self.syntax_kind) self.children = children or [] diff --git a/utils/gyb_syntax_support/StmtNodes.py b/utils/gyb_syntax_support/StmtNodes.py index 2aa824c8230b0..34049bc591679 100644 --- a/utils/gyb_syntax_support/StmtNodes.py +++ b/utils/gyb_syntax_support/StmtNodes.py @@ -91,8 +91,8 @@ is_optional=True), Child('LabelColon', kind='ColonToken', is_optional=True), - Child('For', kind='ForToken'), - Child('Case', kind='CaseToken', + Child('ForKeyword', kind='ForToken'), + Child('CaseKeyword', kind='CaseToken', is_optional=True), Child('ItemPattern', kind='Pattern'), Child('InKeyword', kind='InToken'), diff --git a/utils/gyb_syntax_support/Token.py b/utils/gyb_syntax_support/Token.py index 27388c1499ba2..59ecd29de2d8c 100644 --- a/utils/gyb_syntax_support/Token.py +++ b/utils/gyb_syntax_support/Token.py @@ -1,3 +1,6 @@ +from kinds import lowercase_first_word + + class Token(object): """ Represents the specification for a Token in the TokenSyntax file. @@ -9,6 +12,12 @@ def __init__(self, name, kind, text=None, is_keyword=False): self.text = text or "" self.is_keyword = is_keyword + def swift_kind(self): + name = lowercase_first_word(self.name) + if self.is_keyword: + return name + 'Keyword' + return name + class Keyword(Token): """ @@ -120,6 +129,7 @@ def __init__(self, name, text): Token('InfixQuestionMark', 'question_infix', text='?'), Token('ExclamationMark', 'exclaim_postfix', text='!'), Token('Identifier', 'identifier'), + Token('DollarIdentifier', 'dollarident'), Token('UnspacedBinaryOperator', 'oper_binary_unspaced'), Token('SpacedBinaryOperator', 'oper_binary_spaced'), Token('PrefixOperator', 'oper_prefix'), diff --git a/utils/gyb_syntax_support/__init__.py b/utils/gyb_syntax_support/__init__.py index 38bd175e2d380..36ded629964a8 100644 --- a/utils/gyb_syntax_support/__init__.py +++ b/utils/gyb_syntax_support/__init__.py @@ -20,7 +20,6 @@ def make_missing_child(child): """ Generates a C++ call to make the raw syntax for a given Child object. """ - if child.is_token(): token = child.main_token() tok_kind = "tok::" + token.kind if token else "tok::unknown" @@ -32,6 +31,22 @@ def make_missing_child(child): return 'RawSyntax::missing(SyntaxKind::%s)' % missing_kind +def make_missing_swift_child(child): + """ + Generates a Swift call to make the raw syntax for a given Child object. + """ + if child.is_token(): + token = child.main_token() + tok_kind = token.swift_kind() if token else "unknown" + if token and not token.text: + tok_kind += '("")' + return 'RawSyntax.missingToken(.%s)' % tok_kind + else: + missing_kind = "unknown" if child.syntax_kind == "Syntax" \ + else child.swift_syntax_kind + return 'RawSyntax.missing(.%s)' % missing_kind + + def create_node_map(): """ Creates a lookup table to find nodes by their kind. diff --git a/utils/gyb_syntax_support/kinds.py b/utils/gyb_syntax_support/kinds.py index 0f9920c5832ac..edbf34da0a2e2 100644 --- a/utils/gyb_syntax_support/kinds.py +++ b/utils/gyb_syntax_support/kinds.py @@ -18,3 +18,23 @@ def kind_to_type(kind): if kind.endswith("Token"): return "TokenSyntax" return kind + "Syntax" + + +def lowercase_first_word(name): + """ + Lowercases the first word in the provided camelCase or PascalCase string. + EOF -> eof + IfKeyword -> ifKeyword + EOFToken -> eofToken + """ + word_index = 0 + threshold_index = 1 + for c in name: + if c.islower(): + if word_index > threshold_index: + word_index -= 1 + break + word_index += 1 + if word_index == 0: + return name + return name[:word_index].lower() + name[word_index:] From fd45799e1ff655880fb7f189a3747bd566c9d1b6 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Mon, 14 Aug 2017 12:31:05 -0700 Subject: [PATCH 2/2] Only build SwiftSyntax on macOS --- tools/CMakeLists.txt | 2 +- tools/SwiftSyntax/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d57a7de29e577..a5a29a6ffb37c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -22,7 +22,7 @@ if(SWIFT_HOST_VARIANT STREQUAL "macosx") add_swift_tool_subdirectory(swift-stdlib-tool) if(SWIFT_BUILD_STDLIB) - add_subdirectory(SwiftSyntax) + add_subdirectory(SwiftSyntax) endif() endif() diff --git a/tools/SwiftSyntax/CMakeLists.txt b/tools/SwiftSyntax/CMakeLists.txt index ce594504f0a47..96eff28f6518b 100644 --- a/tools/SwiftSyntax/CMakeLists.txt +++ b/tools/SwiftSyntax/CMakeLists.txt @@ -20,4 +20,5 @@ add_swift_library(swiftSwiftSyntax SHARED SWIFT_MODULE_DEPENDS Foundation INSTALL_IN_COMPONENT swift-syntax + TARGET_SDKS OSX IS_STDLIB)