Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Identifier wrapper that strips backticks from token text #2576

Merged
Merged
56 changes: 56 additions & 0 deletions Sources/SwiftSyntax/Identifier.swift
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add Identifier.swift to CMakeLists.txt. swift-syntax is built using CMake when it’s linked into the compiler.

Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

/// An abstraction for sanitized values on a token.
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
public struct Identifier: Equatable, Hashable, Sendable {
/// The sanitized `text` of a token.
public var name: String {
String(syntaxText: rawIdentifier.name)
}

@_spi(RawSyntax)
public let rawIdentifier: RawIdentifier
plemarquand marked this conversation as resolved.
Show resolved Hide resolved

let arena: RetainedSyntaxArena
plemarquand marked this conversation as resolved.
Show resolved Hide resolved

public init?(_ token: TokenSyntax) {
guard case .identifier(let text) = token.tokenKind else {
return nil
}

var rawText = text.contains("`") ? text.trimmingCharacters(in: "`") : Substring(text)

let syntaxArena = SyntaxArena()

let name = rawText.withUTF8 {
syntaxArena.intern(
SyntaxText(buffer: SyntaxArenaAllocatedBufferPointer<UInt8>($0))
)
}
plemarquand marked this conversation as resolved.
Show resolved Hide resolved

self.rawIdentifier = RawIdentifier(name: name)
self.arena = RetainedSyntaxArena(syntaxArena)
}

public static func == (lhs: Identifier, rhs: Identifier) -> Bool {
lhs.rawIdentifier == rhs.rawIdentifier
}

public func hash(into hasher: inout Hasher) {
hasher.combine(rawIdentifier)
}
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
}

@_spi(RawSyntax)
public struct RawIdentifier: Equatable, Hashable, Sendable {
public let name: SyntaxText
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
}
5 changes: 5 additions & 0 deletions Sources/SwiftSyntax/TokenSyntax.swift
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,11 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
}
}

/// An identifier created from `self`.
public var identifier: Identifier? {
Identifier(self)
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
}

/// A token by itself has no structure, so we represent its structure by an
/// empty layout node.
///
Expand Down
26 changes: 26 additions & 0 deletions Sources/SwiftSyntax/Utils.swift
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,29 @@ extension RawUnexpectedNodesSyntax {
self.init(raw: raw)
}
}

extension String {
func trimmingCharacters(in charactersToTrim: any BidirectionalCollection<Character>) -> Substring {
let leadingCharacters = countOfSequentialCharacters(charactersToTrim, in: self)
let trailingCharacters = countOfSequentialCharacters(charactersToTrim, in: reversed())

return dropFirst(leadingCharacters).dropLast(trailingCharacters)
}
}

private func countOfSequentialCharacters(
_ charactersToCount: any BidirectionalCollection<Character>,
in characters: any BidirectionalCollection<Character>
) -> Int {
var count = 0

for character in characters {
if charactersToCount.contains(character) {
count += 1
} else {
break
}
}

return count
}
51 changes: 51 additions & 0 deletions Tests/SwiftSyntaxTest/IdentifierTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

@_spi(RawSyntax) import SwiftSyntax
import XCTest

class IdentifierTests: XCTestCase {
adammcarter marked this conversation as resolved.
Show resolved Hide resolved
public func testIdentifierInit() {
let someToken = TokenSyntax(stringLiteral: "someToken")
XCTAssertNotNil(Identifier(someToken))

let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
plemarquand marked this conversation as resolved.
Show resolved Hide resolved
XCTAssertNil(Identifier(nonIdentifierToken))
}

public func testName() {
let basicToken = TokenSyntax(stringLiteral: "basicToken")
XCTAssertEqual(Identifier(basicToken)?.name, "basicToken")

let backtickedToken = TokenSyntax(stringLiteral: "`backtickedToken`")
XCTAssertEqual(Identifier(backtickedToken)?.name, "backtickedToken")

let multiBacktickedToken = TokenSyntax(stringLiteral: "```multiBacktickedToken```")
XCTAssertEqual(Identifier(multiBacktickedToken)?.name, "multiBacktickedToken")

let unicodeNormalizedToken = TokenSyntax(stringLiteral: "\u{e0}") // "a`"
XCTAssertEqual(Identifier(unicodeNormalizedToken)?.name, "\u{61}\u{300}") // "à"
}

public func testRawIdentifier() {
let rawIdentifier = TokenSyntax(stringLiteral: "sometoken").identifier?.rawIdentifier
XCTAssertEqual(rawIdentifier?.name, SyntaxText("sometoken"))
}

public func testTokenSyntaxIdentifier() {
let tokenSyntax = TokenSyntax(stringLiteral: "sometoken")
XCTAssertEqual(tokenSyntax.identifier, Identifier(tokenSyntax))

let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
XCTAssertNil(nonIdentifierToken.identifier)
}
}