Skip to content

Commit

Permalink
[Parser] Introduce TokenSpec for fixed text
Browse files Browse the repository at this point in the history 
Matches a specific tokenText. Use it in decl/type attribute
TokenSpecSet.
  • Loading branch information
@rintaro
rintaro committed Feb 8, 2024
1 parent b0b5e46 commit dbcc021
Show file tree
Hide file tree
Showing 5 changed files with 190 additions and 117 deletions.
2 changes: 0 additions & 2 deletions ...tion/Sources/generate-swift-syntax/templates/swiftparser/TokenSpecStaticMembersFile.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,5 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader
for tokenSpec in Token.allCases.map(\.spec) where tokenSpec.kind != .keyword {
DeclSyntax("static var \(tokenSpec.varOrCaseName): TokenSpec { return TokenSpec(.\(tokenSpec.varOrCaseName)) }")
}

DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(keyword) }")
}
}
117 changes: 69 additions & 48 deletions Sources/SwiftParser/Attributes.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,61 +67,82 @@ extension Parser {
case transpose

init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) {
switch lexeme.rawTokenKind {
case .keyword:
switch Keyword(lexeme.tokenText) {
case .rethrows: self = .rethrows
default:
return nil
}
case .identifier:
switch lexeme.tokenText {
case "_alignment": self = ._alignment
case "_backDeploy": self = ._backDeploy
case "_cdecl": self = ._cdecl
case "_documentation": self = ._documentation
case "_dynamicReplacement": self = ._dynamicReplacement
case "_effects": self = ._effects
case "_expose": self = ._expose
case "_implements": self = ._implements
case "_nonSendable": self = ._nonSendable
case "_objcImplementation": self = ._objcImplementation
case "_objcRuntimeName": self = ._objcRuntimeName
case "_optimize": self = ._optimize
case "_originallyDefinedIn": self = ._originallyDefinedIn
case "_private": self = ._private
case "_projectedValueProperty": self = ._projectedValueProperty
case "_semantics": self = ._semantics
case "_specialize": self = ._specialize
case "_spi": self = ._spi
case "_spi_available": self = ._spi_available
case "_swift_native_objc_runtime_base": self = ._swift_native_objc_runtime_base
case "_typeEraser": self = ._typeEraser
case "_unavailableFromAsync": self = ._unavailableFromAsync
case "rethrows": self = .rethrows
case "attached": self = .attached
case "backDeployed": self = .backDeployed
case "derivative": self = .derivative
case "differentiable": self = .differentiable
case "exclusivity": self = .exclusivity
case "freestanding": self = .freestanding
case "inline": self = .inline
case "objc": self = .objc
case "available": self = .available
case "Sendable": self = .Sendable
case "transpose": self = .transpose
default:
return nil
}
switch lexeme {
case TokenSpec("_alignment"): self = ._alignment
case TokenSpec("_backDeploy"): self = ._backDeploy
case TokenSpec("_cdecl"): self = ._cdecl
case TokenSpec("_documentation"): self = ._documentation
case TokenSpec("_dynamicReplacement"): self = ._dynamicReplacement
case TokenSpec("_effects"): self = ._effects
case TokenSpec("_expose"): self = ._expose
case TokenSpec("_implements"): self = ._implements
case TokenSpec("_nonSendable"): self = ._nonSendable
case TokenSpec("_objcImplementation"): self = ._objcImplementation
case TokenSpec("_objcRuntimeName"): self = ._objcRuntimeName
case TokenSpec("_optimize"): self = ._optimize
case TokenSpec("_originallyDefinedIn"): self = ._originallyDefinedIn
case TokenSpec("_private"): self = ._private
case TokenSpec("_projectedValueProperty"): self = ._projectedValueProperty
case TokenSpec("_semantics"): self = ._semantics
case TokenSpec("_specialize"): self = ._specialize
case TokenSpec("_spi"): self = ._spi
case TokenSpec("_spi_available"): self = ._spi_available
case TokenSpec("_swift_native_objc_runtime_base"): self = ._swift_native_objc_runtime_base
case TokenSpec("_typeEraser"): self = ._typeEraser
case TokenSpec("_unavailableFromAsync"): self = ._unavailableFromAsync
case TokenSpec("rethrows"): self = .rethrows
case TokenSpec("attached"): self = .attached
case TokenSpec("available"): self = .available
case TokenSpec("backDeployed"): self = .backDeployed
case TokenSpec("derivative"): self = .derivative
case TokenSpec("differentiable"): self = .differentiable
case TokenSpec("exclusivity"): self = .exclusivity
case TokenSpec("freestanding"): self = .freestanding
case TokenSpec("inline"): self = .inline
case TokenSpec("objc"): self = .objc
case TokenSpec("Sendable"): self = .Sendable
case TokenSpec("transpose"): self = .transpose
default:
return nil
}
}

var spec: TokenSpec {
switch self {
case .rethrows: return .keyword(.rethrows)
default: return .identifier
case ._alignment: return TokenSpec("_alignment")
case ._backDeploy: return TokenSpec("_backDeploy")
case ._cdecl: return TokenSpec("_cdecl")
case ._documentation: return TokenSpec("_documentation")
case ._dynamicReplacement: return TokenSpec("_dynamicReplacement")
case ._effects: return TokenSpec("_effects")
case ._expose: return TokenSpec("_expose")
case ._implements: return TokenSpec("_implements")
case ._nonSendable: return TokenSpec("_nonSendable")
case ._objcImplementation: return TokenSpec("_objcImplementation")
case ._objcRuntimeName: return TokenSpec("_objcRuntimeName")
case ._optimize: return TokenSpec("_optimize")
case ._originallyDefinedIn: return TokenSpec("_originallyDefinedIn")
case ._private: return TokenSpec("_private")
case ._projectedValueProperty: return TokenSpec("_projectedValueProperty")
case ._semantics: return TokenSpec("_semantics")
case ._specialize: return TokenSpec("_specialize")
case ._spi: return TokenSpec("_spi")
case ._spi_available: return TokenSpec("_spi_available")
case ._swift_native_objc_runtime_base: return TokenSpec("_swift_native_objc_runtime_base")
case ._typeEraser: return TokenSpec("_typeEraser")
case ._unavailableFromAsync: return TokenSpec("_unavailableFromAsync")
case .rethrows: return TokenSpec("rethrows")
case .attached: return TokenSpec("attached")
case .available: return TokenSpec("available")
case .backDeployed: return TokenSpec("backDeployed")
case .derivative: return TokenSpec("derivative")
case .differentiable: return TokenSpec("differentiable")
case .exclusivity: return TokenSpec("exclusivity")
case .freestanding: return TokenSpec("freestanding")
case .inline: return TokenSpec("inline")
case .objc: return TokenSpec("objc")
case .Sendable: return TokenSpec("Sendable")
case .transpose: return TokenSpec("transpose")
}
}
}
Expand Down
131 changes: 88 additions & 43 deletions Sources/SwiftParser/TokenSpec.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ struct PrepareForKeywordMatch {
/// The kind of the lexeme.
fileprivate let rawTokenKind: RawTokenKind

fileprivate let rawTokenText: SyntaxText

/// If the lexeme has the same text as a keyword, that keyword, otherwise `nil`.
fileprivate let keyword: Keyword?

Expand All @@ -28,6 +30,7 @@ struct PrepareForKeywordMatch {
@inline(__always)
init(_ lexeme: Lexer.Lexeme) {
self.rawTokenKind = lexeme.rawTokenKind
self.rawTokenText = lexeme.tokenText
switch lexeme.rawTokenKind {
case .keyword, .identifier:
keyword = Keyword(lexeme.tokenText)
Expand All @@ -46,18 +49,24 @@ struct PrepareForKeywordMatch {
/// `matches(rawTokenKind:text:)` based on the matched kind.
@_spi(AlternateTokenIntrospection)
public struct TokenSpec {
/// The kind we expect the token that we want to consume to have.
/// This can be a keyword, in which case the ``TokenSpec`` will also match an
/// identifier with the same text as the keyword and remap it to that keyword
/// when consumed.
///
/// `fileprivate` because only functions in this file should access it since
/// they know how to handle the identifier -> keyword remapping.
fileprivate let rawTokenKind: RawTokenKind
enum Matcher {
/// A token with a specific text.
case fixedText(SyntaxText)

/// If `rawTokenKind` is `keyword`, the keyword we are expecting. For all other
/// values of `rawTokenKind`, this is `nil`.
fileprivate let keyword: Keyword?
/// The keyword we are expecting.
case keyword(Keyword)

/// The kind we expect the token that we want to consume to have.
/// This can be a keyword, in which case the ``TokenSpec`` will also match an
/// identifier with the same text as the keyword and remap it to that keyword
/// when consumed.
///
/// `fileprivate` because only functions in this file should access it since
/// they know how to handle the identifier -> keyword remapping.
case tokenKind(RawTokenKind)
}

fileprivate let matcher: Matcher

/// If not nil, the token will be remapped to the provided kind when consumed.
///
Expand All @@ -83,8 +92,7 @@ public struct TokenSpec {
allowAtStartOfLine: Bool = true
) {
precondition(rawTokenKind != .keyword, "To create a TokenSpec for a keyword use the initializer that takes a keyword")
self.rawTokenKind = rawTokenKind
self.keyword = nil
self.matcher = .tokenKind(rawTokenKind)
self.remapping = remapping
self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(nonKeyword: rawTokenKind)
self.allowAtStartOfLine = allowAtStartOfLine
Expand All @@ -97,39 +105,50 @@ public struct TokenSpec {
recoveryPrecedence: TokenPrecedence? = nil,
allowAtStartOfLine: Bool = true
) {
self.rawTokenKind = .keyword
self.keyword = keyword
self.matcher = .keyword(keyword)
self.remapping = remapping
self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(keyword)
self.allowAtStartOfLine = allowAtStartOfLine
}

@inline(__always)
init(
_ text: SyntaxText,
remapping: RawTokenKind? = nil,
recoveryPrecedence: TokenPrecedence? = nil,
allowAtStartOfLine: Bool = true
) {
self.matcher = .fixedText(text)
self.remapping = remapping
self.recoveryPrecedence = recoveryPrecedence ?? .identifierLike
self.allowAtStartOfLine = allowAtStartOfLine
}

@inline(__always)
func matches(
rawTokenKind: RawTokenKind,
rawTokenText: SyntaxText,
keyword: @autoclosure () -> Keyword?,
atStartOfLine: @autoclosure () -> Bool
) -> Bool {
if !allowAtStartOfLine && atStartOfLine() {
return false
}
if self.rawTokenKind == .keyword {
precondition(self.keyword != nil)
switch rawTokenKind {
case .keyword, .identifier:
return keyword() == self.keyword
default:
return false
}
} else {
return rawTokenKind == self.rawTokenKind
switch matcher {
case .fixedText(let expectedText):
return rawTokenText == expectedText
case .keyword(let expectedKeyword):
return keyword() == expectedKeyword
case .tokenKind(let expectedRawTokenKind):
return rawTokenKind == expectedRawTokenKind
}
}

@inline(__always)
static func ~= (kind: TokenSpec, lexeme: Lexer.Lexeme) -> Bool {
return kind.matches(
rawTokenKind: lexeme.rawTokenKind,
rawTokenText: lexeme.tokenText,
keyword: Keyword(lexeme.tokenText),
atStartOfLine: lexeme.isAtStartOfLine
)
Expand All @@ -138,7 +157,8 @@ public struct TokenSpec {
@inline(__always)
static func ~= (kind: TokenSpec, token: TokenSyntax) -> Bool {
return kind.matches(
rawTokenKind: token.tokenView.rawKind,
rawTokenKind: token.rawTokenKind,
rawTokenText: token.rawText,
keyword: Keyword(token.tokenView.rawText),
atStartOfLine: token.leadingTrivia.contains(where: { $0.isNewline })
)
Expand All @@ -148,6 +168,7 @@ public struct TokenSpec {
static func ~= (kind: TokenSpec, token: RawTokenSyntax) -> Bool {
return kind.matches(
rawTokenKind: token.tokenKind,
rawTokenText: token.tokenText,
keyword: Keyword(token.tokenView.rawText),
atStartOfLine: token.leadingTriviaPieces.contains(where: \.isNewline)
)
Expand All @@ -157,6 +178,7 @@ public struct TokenSpec {
static func ~= (kind: TokenSpec, lexeme: PrepareForKeywordMatch) -> Bool {
return kind.matches(
rawTokenKind: lexeme.rawTokenKind,
rawTokenText: lexeme.rawTokenText,
keyword: lexeme.keyword,
atStartOfLine: lexeme.isAtStartOfLine
)
Expand All @@ -169,29 +191,50 @@ public struct TokenSpec {
/// modification of test cases. This should never be used in the parser itself.
@_spi(AlternateTokenIntrospection)
public var synthesizedTokenKind: TokenKind {
switch rawTokenKind {
case .binaryOperator: return .binaryOperator("+")
case .dollarIdentifier: return .dollarIdentifier("$0")
case .floatLiteral: return .floatLiteral("1.0")
case .identifier: return .identifier("myIdent")
case .integerLiteral: return .integerLiteral("1")
case .keyword: return .keyword(keyword!)
case .postfixOperator: return .postfixOperator("++")
case .prefixOperator: return .prefixOperator("!")
case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#")
case .regexLiteralPattern: return .regexLiteralPattern(".*")
case .regexPoundDelimiter: return .regexPoundDelimiter("#")
case .stringSegment: return .stringSegment("abc")
default: return TokenKind.fromRaw(kind: rawTokenKind, text: "")
switch matcher {
case .fixedText(let text):
return .identifier(String(syntaxText: text))
case .keyword(let keyword):
return .keyword(keyword)
case .tokenKind(let kind):
switch kind {
case .binaryOperator: return .binaryOperator("+")
case .dollarIdentifier: return .dollarIdentifier("$0")
case .floatLiteral: return .floatLiteral("1.0")
case .identifier: return .identifier("myIdent")
case .integerLiteral: return .integerLiteral("1")
case .postfixOperator: return .postfixOperator("++")
case .prefixOperator: return .prefixOperator("!")
case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#")
case .regexLiteralPattern: return .regexLiteralPattern(".*")
case .regexPoundDelimiter: return .regexPoundDelimiter("#")
case .stringSegment: return .stringSegment("abc")
default: return TokenKind.fromRaw(kind: kind, text: "")
}
}
}

static func keyword(_ keyword: Keyword) -> TokenSpec {
return TokenSpec(keyword)
}

static func fixedText(_ text: SyntaxText) -> TokenSpec {
return TokenSpec(text)
}
}

extension TokenConsumer {
/// Generates a missing token that has the expected kind of `spec`.
@inline(__always)
mutating func missingToken(_ spec: TokenSpec) -> Token {
return missingToken(spec.remapping ?? spec.rawTokenKind, text: spec.keyword?.defaultText ?? spec.rawTokenKind.defaultText)
switch spec.matcher {
case .fixedText(let text):
return missingToken(spec.remapping ?? .identifier, text: text)
case .keyword(let keyword):
return missingToken(spec.remapping ?? .keyword, text: keyword.defaultText)
case .tokenKind(let kind):
return missingToken(spec.remapping ?? kind, text: kind.defaultText)
}
}

/// Asserts that the current token matches `spec` and consumes it, performing
Expand All @@ -204,9 +247,11 @@ extension TokenConsumer {
precondition(spec ~= self.currentToken)
if let remapping = spec.remapping {
return self.consumeAnyToken(remapping: remapping)
} else if spec.rawTokenKind == .keyword {
}
switch spec.matcher {
case .keyword(_):
return self.consumeAnyToken(remapping: .keyword)
} else {
case .fixedText(_), .tokenKind(_):
return self.consumeAnyToken()
}
}
Expand Down

0 comments on commit dbcc021

Please sign in to comment.