From 5a831c4b5aae5c8a0fb5a5463e2cb451d73d9888 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sun, 21 Jan 2024 13:27:10 -0600 Subject: [PATCH 1/9] Rework Async tree-sitter Model --- Package.resolved | 26 +- .../CodeEditSourceEditor.swift | 2 +- .../TextViewController+Highlighter.swift | 8 +- .../Controller/TextViewController.swift | 12 +- .../Enums/CaptureName.swift | 2 +- .../HighlighterTextView+createReadBlock.swift | 17 +- .../NSRange+/NSRange+InputEdit.swift | 23 +- .../Extensions/Parser+createTree.swift | 20 -- .../Extensions/TextView+/TextView+Point.swift | 18 ++ .../Highlighting/HighlightProviding.swift | 37 +-- .../Highlighting/HighlightRange.swift | 9 +- .../Highlighter+NSTextStorageDelegate.swift | 44 +++ .../Highlighting/Highlighter.swift | 274 +++++++++++------- .../Highlighting/HighlighterTextView.swift | 10 +- .../TreeSitter/LanguageLayer.swift | 52 ++-- .../TreeSitter/PthreadLock.swift | 33 --- .../TreeSitter/TreeSitterClient+Edit.swift | 86 ++---- .../TreeSitterClient+Highlight.swift | 71 ++--- .../TreeSitter/TreeSitterClient.swift | 234 ++++----------- .../TreeSitter/TreeSitterState.swift | 49 ++-- .../TreeSitterClientTests.swift | 182 ++++++------ 21 files changed, 551 insertions(+), 658 deletions(-) delete mode 100644 Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift create mode 100644 Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift create mode 100644 Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift delete mode 100644 Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift diff --git a/Package.resolved b/Package.resolved index 7e2a1391e..de9cb3ae1 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,23 +1,5 @@ { "pins" : [ - { - "identity" : "codeeditlanguages", - "kind" : "remoteSourceControl", - "location" : "https://github.com/CodeEditApp/CodeEditLanguages.git", - "state" : { - "revision" : "af29ab4a15474a0a38ef88ef65c20e58a0812e43", - "version" : "0.1.17" - } - }, - { - "identity" : "codeedittextview", - "kind" : "remoteSourceControl", - "location" : "https://github.com/CodeEditApp/CodeEditTextView.git", - "state" : { - "revision" : "c867fed329b2b4ce91a13742e20626f50cf233bb", - "version" : "0.7.0" - } - }, { "identity" : "mainoffender", "kind" : "remoteSourceControl", @@ -41,8 +23,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "a902f1823a7ff3c9ab2fba0f992396b948eda307", - "version" : "1.0.5" + "revision" : "d029d9d39c87bed85b1c50adee7c41795261a192", + "version" : "1.0.6" } }, { @@ -59,8 +41,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ChimeHQ/SwiftTreeSitter", "state" : { - "revision" : "df25a52f72ebc5b50ae20d26d1363793408bb28b", - "version" : "0.7.1" + "revision" : "2599e95310b3159641469d8a21baf2d3d200e61f", + "version" : "0.8.0" } }, { diff --git a/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift b/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift index 7308efa6c..446ec7a8b 100644 --- a/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift +++ b/Sources/CodeEditSourceEditor/CodeEditSourceEditor.swift @@ -212,7 +212,7 @@ public struct CodeEditSourceEditor: NSViewControllerRepresentable { @MainActor public class Coordinator: NSObject { var parent: CodeEditSourceEditor - var controller: TextViewController? + weak var controller: TextViewController? var isUpdatingFromRepresentable: Bool = false var isUpdateFromTextView: Bool = false diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift index e56a2292a..c64a49b7f 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift @@ -12,6 +12,7 @@ extension TextViewController { internal func setUpHighlighter() { if let highlighter { textView.removeStorageDelegate(highlighter) + highlighter.cancelAllTasks() self.highlighter = nil } @@ -32,11 +33,8 @@ extension TextViewController { if let highlightProvider = highlightProvider { provider = highlightProvider } else { - let textProvider: ResolvingQueryCursor.TextProvider = { [weak self] range, _ -> String? in - return self?.textView.textStorage.mutableString.substring(with: range) - } - - provider = TreeSitterClient(textProvider: textProvider) + self.treeSitterClient = TreeSitterClient() + provider = self.treeSitterClient! } if let provider = provider { diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift index 69d4e59ca..ee74a1710 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift @@ -149,7 +149,12 @@ public class TextViewController: NSViewController { } } - internal var highlighter: Highlighter? + var highlighter: Highlighter? + + /// The tree sitter client managed by the source editor. + /// + /// This will be `nil` if another highlighter provider is passed to the source editor. + internal(set) public var treeSitterClient: TreeSitterClient? private var fontCharWidth: CGFloat { (" " as NSString).size(withAttributes: [.font: font]).width } @@ -292,6 +297,11 @@ public class TextViewController: NSViewController { deinit { if let highlighter { textView.removeStorageDelegate(highlighter) + Task { + // We can safely do this async operation here b/c the highlighter will not deinit until its + // tasks are finished. + await highlighter.cancelAllTasks() + } } highlighter = nil highlightProvider = nil diff --git a/Sources/CodeEditSourceEditor/Enums/CaptureName.swift b/Sources/CodeEditSourceEditor/Enums/CaptureName.swift index 941d1b6a5..b73a9a251 100644 --- a/Sources/CodeEditSourceEditor/Enums/CaptureName.swift +++ b/Sources/CodeEditSourceEditor/Enums/CaptureName.swift @@ -6,7 +6,7 @@ // /// A collection of possible capture names for `tree-sitter` with their respected raw values. -public enum CaptureName: String, CaseIterable { +public enum CaptureName: String, CaseIterable, Sendable { case include case constructor case keyword diff --git a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift b/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift index ac5976d0a..c0d58fc80 100644 --- a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift +++ b/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift @@ -6,20 +6,27 @@ // import Foundation +import CodeEditTextView import SwiftTreeSitter -extension HighlighterTextView { +extension TextView { func createReadBlock() -> Parser.ReadBlock { - return { byteOffset, _ in - let limit = self.documentRange.length + return { [weak self] byteOffset, _ in + let limit = self?.documentRange.length ?? 0 let location = byteOffset / 2 let end = min(location + (1024), limit) - if location > end { + if location > end || self == nil { // Ignore and return nothing, tree-sitter's internal tree can be incorrect in some situations. return nil } let range = NSRange(location.. SwiftTreeSitter.Predicate.TextProvider { + return { range, _ in + return self.stringForRange(range) } } } diff --git a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift index 7252541e1..1ef871771 100644 --- a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift @@ -6,10 +6,11 @@ // import Foundation +import CodeEditTextView import SwiftTreeSitter extension InputEdit { - init?(range: NSRange, delta: Int, oldEndPoint: Point) { + init?(range: NSRange, delta: Int, oldEndPoint: Point, textView: TextView) { let newEndLocation = NSMaxRange(range) + delta if newEndLocation < 0 { @@ -17,16 +18,18 @@ extension InputEdit { return nil } - // TODO: - Ask why Neon only uses .zero for these - let startPoint: Point = .zero - let newEndPoint: Point = .zero + let newRange = NSRange(location: range.location, length: range.length + delta) + let startPoint = textView.pointForLocation(range.location) ?? .zero + let newEndPoint = textView.pointForLocation(newEndLocation) ?? .zero - self.init(startByte: UInt32(range.location * 2), - oldEndByte: UInt32(NSMaxRange(range) * 2), - newEndByte: UInt32(newEndLocation * 2), - startPoint: startPoint, - oldEndPoint: oldEndPoint, - newEndPoint: newEndPoint) + self.init( + startByte: UInt32(range.location * 2), + oldEndByte: UInt32(NSMaxRange(range) * 2), + newEndByte: UInt32(newEndLocation * 2), + startPoint: startPoint, + oldEndPoint: oldEndPoint, + newEndPoint: newEndPoint + ) } } diff --git a/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift b/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift deleted file mode 100644 index 514c5b5d7..000000000 --- a/Sources/CodeEditSourceEditor/Extensions/Parser+createTree.swift +++ /dev/null @@ -1,20 +0,0 @@ -// -// Parser+createTree.swift -// CodeEditSourceEditor -// -// Created by Khan Winter on 5/20/23. -// - -import Foundation -import SwiftTreeSitter - -extension Parser { - /// Creates a tree-sitter tree. - /// - Parameters: - /// - parser: The parser object to use to parse text. - /// - readBlock: A callback for fetching blocks of text. - /// - Returns: A tree if it could be parsed. - internal func createTree(readBlock: @escaping Parser.ReadBlock) -> Tree? { - return parse(tree: nil, readBlock: readBlock) - } -} diff --git a/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift new file mode 100644 index 000000000..3008ae489 --- /dev/null +++ b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+Point.swift @@ -0,0 +1,18 @@ +// +// TextView+Point.swift +// CodeEditSourceEditor +// +// Created by Khan Winter on 1/18/24. +// + +import Foundation +import CodeEditTextView +import SwiftTreeSitter + +extension TextView { + func pointForLocation(_ location: Int) -> Point? { + guard let linePosition = layoutManager.textLineForOffset(location) else { return nil } + let column = location - linePosition.range.location + return Point(row: linePosition.index, column: column) + } +} diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift index ec8fe89f1..56ed660af 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift @@ -6,42 +6,43 @@ // import Foundation +import CodeEditTextView import CodeEditLanguages import AppKit /// The protocol a class must conform to to be used for highlighting. public protocol HighlightProviding: AnyObject { - /// A unique identifier for the highlighter object. - /// Example: `"CodeEdit.TreeSitterHighlighter"` - /// - Note: This does not need to be *globally* unique, merely unique across all the highlighters used. - var identifier: String { get } - /// Called once to set up the highlight provider with a data source and language. /// - Parameters: /// - textView: The text view to use as a text source. - /// - codeLanguage: The langugage that should be used by the highlighter. - func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) + /// - codeLanguage: The language that should be used by the highlighter. + func setUp(textView: TextView, codeLanguage: CodeLanguage) async + + /// Notifies the highlighter that an edit is going to happen in the given range. + /// - Parameters: + /// - textView: The text view to use. + /// - range: The range of the incoming edit. + func willApplyEdit(textView: TextView, range: NSRange) async /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. /// - Parameters: - /// - textView:The text view to use. + /// - textView: The text view to use. /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. - /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - func applyEdit(textView: HighlighterTextView, - range: NSRange, - delta: Int, - completion: @escaping ((IndexSet) -> Void)) + /// - Returns: an `IndexSet` containing all Indices to invalidate. + func applyEdit(textView: TextView, range: NSRange, delta: Int) async -> IndexSet /// Queries the highlight provider for any ranges to apply highlights to. The highlight provider should return an /// array containing all ranges to highlight, and the capture type for the range. Any ranges or indexes /// excluded from the returned array will be treated as plain text and highlighted as such. /// - Parameters: /// - textView: The text view to use. - /// - range: The range to operate on. - /// - completion: Function to call with all ranges to highlight - func queryHighlightsFor(textView: HighlighterTextView, - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void)) + /// - range: The range to query. + /// - Returns: All highlight ranges for the queried ranges. + func queryHighlightsFor(textView: TextView, range: NSRange) async -> [HighlightRange] +} + +extension HighlightProviding { + public func willApplyEdit(textView: TextView, range: NSRange) async { } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift index 710e206f0..ffb2837fd 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightRange.swift @@ -7,13 +7,8 @@ import Foundation -/// This class represents a range to highlight, as well as the capture name for syntax coloring. -public class HighlightRange { - init(range: NSRange, capture: CaptureName?) { - self.range = range - self.capture = capture - } - +/// This struct represents a range to highlight, as well as the capture name for syntax coloring. +public struct HighlightRange: Sendable { let range: NSRange let capture: CaptureName? } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift new file mode 100644 index 000000000..0edd3fcd5 --- /dev/null +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift @@ -0,0 +1,44 @@ +// +// Highlighter+NSTextStorageDelegate.swift +// CodeEditSourceEditor +// +// Created by Khan Winter on 1/18/24. +// + +import AppKit + +extension Highlighter: NSTextStorageDelegate { + /// Processes an edited range in the text. + /// Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. + func textStorage( + _ textStorage: NSTextStorage, + didProcessEditing editedMask: NSTextStorageEditActions, + range editedRange: NSRange, + changeInLength delta: Int + ) { + // This method is called whenever attributes are updated, so to avoid re-highlighting the entire document + // each time an attribute is applied, we check to make sure this is in response to an edit. + guard editedMask.contains(.editedCharacters) else { return } + + let editTask = Task { + await storageDidEdit(editedRange: editedRange, delta: delta) + return // To keep the compiler happy + } + runningTasks.insert(editTask) + } + + func textStorage( + _ textStorage: NSTextStorage, + willProcessEditing editedMask: NSTextStorageEditActions, + range editedRange: NSRange, + changeInLength delta: Int + ) { + guard editedMask.contains(.editedCharacters) else { return } + + let editTask = Task { + await storageWillEdit(editedRange: editedRange) + return; + } + runningTasks.insert(editTask) + } +} diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift index 9277b16d5..eb3c26ed9 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift @@ -18,6 +18,7 @@ import CodeEditLanguages /// memory and it will listen for bounds changes, text changes, etc. However, to completely invalidate all /// highlights use the ``invalidate()`` method to re-highlight all (visible) text, and the ``setLanguage`` /// method to update the highlighter with a new language if needed. +@MainActor class Highlighter: NSObject { // MARK: - Index Sets @@ -30,20 +31,19 @@ class Highlighter: NSObject { /// The set of valid indexes private var validSet: IndexSet = .init() - /// The range of the entire document - private var entireTextRange: Range { - return 0..<(textView.textStorage.length) - } - /// The set of visible indexes in tht text view lazy private var visibleSet: IndexSet = { - return IndexSet(integersIn: textView.visibleTextRange ?? NSRange()) + return IndexSet(integersIn: textView?.visibleTextRange ?? NSRange()) }() + // MARK: - Tasks + + var runningTasks: Set> = [] + // MARK: - UI /// The text view to highlight - private unowned var textView: TextView + private weak var textView: TextView? /// The editor theme private var theme: EditorTheme @@ -55,10 +55,10 @@ class Highlighter: NSObject { private var language: CodeLanguage /// Calculates invalidated ranges given an edit. - private weak var highlightProvider: HighlightProviding? + private(set) weak var highlightProvider: HighlightProviding? /// The length to chunk ranges into when passing to the highlighter. - fileprivate let rangeChunkLimit = 256 + private let rangeChunkLimit = 256 // MARK: - Init @@ -82,18 +82,26 @@ class Highlighter: NSObject { super.init() - highlightProvider?.setUp(textView: textView, codeLanguage: language) + let setupTask = Task { + await highlightProvider?.setUp(textView: textView, codeLanguage: language) + return + } + runningTasks.insert(setupTask) if let scrollView = textView.enclosingScrollView { - NotificationCenter.default.addObserver(self, - selector: #selector(visibleTextChanged(_:)), - name: NSView.frameDidChangeNotification, - object: scrollView) - - NotificationCenter.default.addObserver(self, - selector: #selector(visibleTextChanged(_:)), - name: NSView.boundsDidChangeNotification, - object: scrollView.contentView) + NotificationCenter.default.addObserver( + self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.frameDidChangeNotification, + object: scrollView + ) + + NotificationCenter.default.addObserver( + self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.boundsDidChangeNotification, + object: scrollView.contentView + ) } } @@ -101,27 +109,56 @@ class Highlighter: NSObject { /// Invalidates all text in the textview. Useful for updating themes. public func invalidate() { - updateVisibleSet() - invalidate(range: NSRange(entireTextRange)) + guard let textView else { return } + updateVisibleSet(textView: textView) + invalidate(range: textView.documentRange) } /// Sets the language and causes a re-highlight of the entire text. /// - Parameter language: The language to update to. public func setLanguage(language: CodeLanguage) { - highlightProvider?.setUp(textView: textView, codeLanguage: language) - invalidate() + cancelAllTasks() + + let setLanguageTask = Task { + guard let textView else { return } + await highlightProvider?.setUp(textView: textView, codeLanguage: language) + guard !Task.isCancelled else { return } + invalidate() + } + runningTasks.insert(setLanguageTask) } /// Sets the highlight provider. Will cause a re-highlight of the entire text. /// - Parameter provider: The provider to use for future syntax highlights. public func setHighlightProvider(_ provider: HighlightProviding) { - self.highlightProvider = provider - highlightProvider?.setUp(textView: textView, codeLanguage: language) - invalidate() + cancelAllTasks() + + highlightProvider = provider + let setHighlightProviderTask = Task { + guard let textView else { return } + await highlightProvider?.setUp(textView: textView, codeLanguage: language) + guard !Task.isCancelled else { return } + invalidate() + } + runningTasks.insert(setHighlightProviderTask) + } + + func cancelAllTasks() { + for task in runningTasks { + task.cancel() + } + runningTasks.removeAll() } deinit { + print("Highlighter deinit") + for task in runningTasks { + task.cancel() + } + runningTasks.removeAll() self.attributeProvider = nil + self.textView = nil + self.highlightProvider = nil } } @@ -140,77 +177,102 @@ private extension Highlighter { validSet.subtract(set) - highlightNextRange() + highlightInvalidRanges() } /// Begins highlighting any invalid ranges - func highlightNextRange() { + func highlightInvalidRanges() { // If there aren't any more ranges to highlight, don't do anything, otherwise continue highlighting // any available ranges. - guard let range = getNextRange() else { - return + var rangesToQuery: [NSRange] = [] + while let range = getNextRange() { + rangesToQuery.append(range) + pendingSet.insert(range: range) } - highlight(range: range) - - highlightNextRange() + queryHighlights(for: rangesToQuery) } - /// Highlights the given range - /// - Parameter range: The range to request highlights for. - func highlight(range rangeToHighlight: NSRange) { - pendingSet.insert(integersIn: rangeToHighlight) - - highlightProvider?.queryHighlightsFor( - textView: self.textView, - range: rangeToHighlight - ) { [weak self] highlightRanges in - guard let attributeProvider = self?.attributeProvider, - let textView = self?.textView else { return } - - self?.pendingSet.remove(integersIn: rangeToHighlight) - guard self?.visibleSet.intersects(integersIn: rangeToHighlight) ?? false else { - return + /// Highlights the given ranges + /// - Parameter ranges: The ranges to request highlights for. + func queryHighlights(for rangesToHighlight: [NSRange]) { + for range in rangesToHighlight { + pendingSet.insert(integersIn: range) + } + + let queryTask = Task.detached { + await withTaskGroup(of: Void.self) { group in + for range in rangesToHighlight { + group.addTask { [weak self] in + guard let textView = await self?.textView else { return } + + let highlights = await self?.highlightProvider?.queryHighlightsFor( + textView: textView, + range: range + ) + + guard !Task.isCancelled else { return } + + await self?.applyHighlightResult(highlights ?? [], rangeToHighlight: range) + } + } } - self?.validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) + } + runningTasks.insert(queryTask) + } + + /// Applies a highlight query result to the text view. + /// - Parameters: + /// - results: The result of a highlight query. + /// - rangeToHighlight: The range to apply the highlight to. + @MainActor + private func applyHighlightResult(_ results: [HighlightRange], rangeToHighlight: NSRange) { + guard let attributeProvider = self.attributeProvider else { + return + } - // Loop through each highlight and modify the textStorage accordingly. - textView.layoutManager.beginTransaction() - textView.textStorage.beginEditing() + pendingSet.remove(integersIn: rangeToHighlight) + guard visibleSet.intersects(integersIn: rangeToHighlight) else { + return + } + validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) - // Create a set of indexes that were not highlighted. - var ignoredIndexes = IndexSet(integersIn: rangeToHighlight) + // Loop through each highlight and modify the textStorage accordingly. + textView?.layoutManager.beginTransaction() + textView?.textStorage.beginEditing() - // Apply all highlights that need color - for highlight in highlightRanges { - textView.textStorage.setAttributes( - attributeProvider.attributesFor(highlight.capture), - range: highlight.range - ) + // Create a set of indexes that were not highlighted. + var ignoredIndexes = IndexSet(integersIn: rangeToHighlight) - // Remove highlighted indexes from the "ignored" indexes. - ignoredIndexes.remove(integersIn: highlight.range) - } + // Apply all highlights that need color + for highlight in results { + textView?.textStorage.setAttributes( + attributeProvider.attributesFor(highlight.capture), + range: highlight.range + ) - // For any indices left over, we need to apply normal attributes to them - // This fixes the case where characters are changed to have a non-text color, and then are skipped when - // they need to be changed back. - for ignoredRange in ignoredIndexes.rangeView { - textView.textStorage.setAttributes( - attributeProvider.attributesFor(nil), - range: NSRange(ignoredRange) - ) - } + // Remove highlighted indexes from the "ignored" indexes. + ignoredIndexes.remove(integersIn: highlight.range) + } - textView.textStorage.endEditing() - textView.layoutManager.endTransaction() + // For any indices left over, we need to apply normal attributes to them + // This fixes the case where characters are changed to have a non-text color, and then are skipped when + // they need to be changed back. + for ignoredRange in ignoredIndexes.rangeView { + textView?.textStorage.setAttributes( + attributeProvider.attributesFor(nil), + range: NSRange(ignoredRange) + ) } + + textView?.textStorage.endEditing() + textView?.layoutManager.endTransaction() } /// Gets the next `NSRange` to highlight based on the invalid set, visible set, and pending set. /// - Returns: An `NSRange` to highlight if it could be fetched. func getNextRange() -> NSRange? { - let set: IndexSet = IndexSet(integersIn: entireTextRange) // All text + let set: IndexSet = IndexSet(integersIn: textView?.documentRange ?? .zero) // All text .subtracting(validSet) // Subtract valid = Invalid set .intersection(visibleSet) // Only visible indexes .subtracting(pendingSet) // Don't include pending indexes @@ -220,8 +282,10 @@ private extension Highlighter { } // Chunk the ranges in sets of rangeChunkLimit characters. - return NSRange(location: range.lowerBound, - length: min(rangeChunkLimit, range.upperBound - range.lowerBound)) + return NSRange( + location: range.lowerBound, + length: min(rangeChunkLimit, range.upperBound - range.lowerBound) + ) } } @@ -229,7 +293,7 @@ private extension Highlighter { // MARK: - Visible Content Updates private extension Highlighter { - private func updateVisibleSet() { + private func updateVisibleSet(textView: TextView) { if let newVisibleRange = textView.visibleTextRange { visibleSet = IndexSet(integersIn: newVisibleRange) } @@ -237,7 +301,11 @@ private extension Highlighter { /// Updates the view to highlight newly visible text when the textview is scrolled or bounds change. @objc func visibleTextChanged(_ notification: Notification) { - updateVisibleSet() + guard let clipView = notification.object as? NSClipView, + let textView = clipView.enclosingScrollView?.documentView as? TextView else { + return + } + updateVisibleSet(textView: textView) // Any indices that are both *not* valid and in the visible text range should be invalidated let newlyInvalidSet = visibleSet.subtracting(validSet) @@ -248,37 +316,37 @@ private extension Highlighter { } } -// MARK: - NSTextStorageDelegate - -extension Highlighter: NSTextStorageDelegate { - /// Processes an edited range in the text. - /// Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. - func textStorage(_ textStorage: NSTextStorage, - didProcessEditing editedMask: NSTextStorageEditActions, - range editedRange: NSRange, - changeInLength delta: Int) { - // This method is called whenever attributes are updated, so to avoid re-highlighting the entire document - // each time an attribute is applied, we check to make sure this is in response to an edit. - guard editedMask.contains(.editedCharacters) else { - return - } +// MARK: - Editing + +extension Highlighter { + func storageDidEdit(editedRange: NSRange, delta: Int) async { + guard let textView else { return } let range = NSRange(location: editedRange.location, length: editedRange.length - delta) if delta > 0 { visibleSet.insert(range: editedRange) } - highlightProvider?.applyEdit(textView: self.textView, - range: range, - delta: delta) { [weak self] invalidatedIndexSet in - let indexSet = invalidatedIndexSet - .union(IndexSet(integersIn: editedRange)) - // Only invalidate indices that are visible. - .intersection(self?.visibleSet ?? .init()) + guard let invalidatedIndexSet = await highlightProvider?.applyEdit( + textView: textView, + range: range, + delta: delta + ) else { + return + } + + let indexSet = invalidatedIndexSet + .union(IndexSet(integersIn: editedRange)) + // Only invalidate indices that are visible. + .intersection(visibleSet) - for range in indexSet.rangeView { - self?.invalidate(range: NSRange(range)) - } + for range in indexSet.rangeView { + invalidate(range: NSRange(range)) } } + + func storageWillEdit(editedRange: NSRange) async { + guard let textView else { return } + await highlightProvider?.willApplyEdit(textView: textView, range: editedRange) + } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift index ed8b4e355..e4e8930e2 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlighterTextView.swift @@ -9,15 +9,7 @@ import Foundation import AppKit import CodeEditTextView -/// The object `HighlightProviding` objects are given when asked for highlights. -public protocol HighlighterTextView: AnyObject { - /// The entire range of the document. - var documentRange: NSRange { get } - /// A substring for the requested range. - func stringForRange(_ nsRange: NSRange) -> String? -} - -extension TextView: HighlighterTextView { +extension TextView { public func stringForRange(_ nsRange: NSRange) -> String? { textStorage.substring(from: nsRange) } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift index 8054a6290..39205b8d4 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift @@ -22,7 +22,7 @@ public class LanguageLayer: Hashable { id: TreeSitterLanguage, parser: Parser, supportsInjections: Bool, - tree: Tree? = nil, + tree: MutableTree? = nil, languageQuery: Query? = nil, ranges: [NSRange] ) { @@ -39,7 +39,7 @@ public class LanguageLayer: Hashable { let id: TreeSitterLanguage let parser: Parser let supportsInjections: Bool - var tree: Tree? + var tree: MutableTree? var languageQuery: Query? var ranges: [NSRange] @@ -48,7 +48,7 @@ public class LanguageLayer: Hashable { id: id, parser: parser, supportsInjections: supportsInjections, - tree: tree?.copy(), + tree: tree?.mutableCopy(), languageQuery: languageQuery, ranges: ranges ) @@ -65,38 +65,36 @@ public class LanguageLayer: Hashable { /// Calculates a series of ranges that have been invalidated by a given edit. /// - Parameters: - /// - textView: The text view to use for text. /// - edit: The edit to act on. /// - timeout: The maximum time interval the parser can run before halting. /// - readBlock: A callback for fetching blocks of text. /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. func findChangedByteRanges( - textView: HighlighterTextView, edit: InputEdit, timeout: TimeInterval?, readBlock: @escaping Parser.ReadBlock ) throws -> [NSRange] { parser.timeout = timeout ?? 0 - let (oldTree, newTree) = calculateNewState( + let newTree = calculateNewState( tree: self.tree, parser: self.parser, edit: edit, readBlock: readBlock ) - if oldTree == nil && newTree == nil { + if self.tree == nil && newTree == nil { // There was no existing tree, make a new one and return all indexes. - tree = parser.createTree(readBlock: readBlock) - return [NSRange(textView.documentRange.intRange)] - } else if oldTree != nil && newTree == nil { + self.tree = parser.parse(tree: nil as Tree?, readBlock: readBlock) + return [self.tree?.rootNode?.range ?? .zero] + } else if self.tree != nil && newTree == nil { // The parser timed out, throw Error.parserTimeout } - let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } + let ranges = changedByteRanges(self.tree, newTree).map { $0.range } - tree = newTree + self.tree = newTree return ranges } @@ -110,21 +108,26 @@ public class LanguageLayer: Hashable { /// - readBlock: The block to use to read text. /// - Returns: (The old state, the new state). internal func calculateNewState( - tree: Tree?, + tree: MutableTree?, parser: Parser, edit: InputEdit, readBlock: @escaping Parser.ReadBlock - ) -> (Tree?, Tree?) { - guard let oldTree = tree else { - return (nil, nil) + ) -> MutableTree? { + guard let tree else { + return nil } // Apply the edit to the old tree - oldTree.edit(edit) + tree.edit(edit) - let newTree = parser.parse(tree: oldTree, readBlock: readBlock) + // Check every timeout to see if the task is canceled to avoid parsing after the editor has been closed. + // We can continue a parse after a timeout causes it to cancel by calling parse on the same tree. + var newTree: MutableTree? = nil + while newTree == nil && !Task.isCancelled { + newTree = parser.parse(tree: tree, readBlock: readBlock) + } - return (oldTree.copy(), newTree) + return newTree } /// Calculates the changed byte ranges between two trees. @@ -132,7 +135,7 @@ public class LanguageLayer: Hashable { /// - lhs: The first (older) tree. /// - rhs: The second (newer) tree. /// - Returns: Any changed ranges. - internal func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { + internal func changedByteRanges(_ lhs: MutableTree?, _ rhs: MutableTree?) -> [Range] { switch (lhs, rhs) { case (let tree1?, let tree2?): return tree1.changedRanges(from: tree2).map({ $0.bytes }) @@ -145,7 +148,14 @@ public class LanguageLayer: Hashable { } } - enum Error: Swift.Error { + enum Error: Swift.Error, LocalizedError { case parserTimeout + + var localizedDescription: String { + switch self { + case .parserTimeout: + return "Parser Timed Out." + } + } } } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift b/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift deleted file mode 100644 index c34e3fd42..000000000 --- a/Sources/CodeEditSourceEditor/TreeSitter/PthreadLock.swift +++ /dev/null @@ -1,33 +0,0 @@ -// -// PthreadLock.swift -// CodeEditSourceEditor -// -// Created by Khan Winter on 6/2/23. -// - -import Foundation - -/// A thread safe, atomic lock that wraps a `pthread_mutex_t` -class PthreadLock { - private var _lock: pthread_mutex_t - - /// Initializes the lock - init() { - _lock = .init() - pthread_mutex_init(&_lock, nil) - } - - /// Locks the lock, if the lock is already locked it will block the current thread until it unlocks. - func lock() { - pthread_mutex_lock(&_lock) - } - - /// Unlocks the lock. - func unlock() { - pthread_mutex_unlock(&_lock) - } - - deinit { - pthread_mutex_destroy(&_lock) - } -} diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift index fae473040..ba212b99e 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient+Edit.swift @@ -10,102 +10,54 @@ import SwiftTreeSitter import CodeEditLanguages extension TreeSitterClient { - /// This class contains an edit state that can be resumed if a parser hits a timeout. - class EditState { - var edit: InputEdit - var rangeSet: IndexSet - var layerSet: Set - var touchedLayers: Set - var completion: ((IndexSet) -> Void) - - init( - edit: InputEdit, - minimumCapacity: Int = 0, - completion: @escaping (IndexSet) -> Void - ) { - self.edit = edit - self.rangeSet = IndexSet() - self.layerSet = Set(minimumCapacity: minimumCapacity) - self.touchedLayers = Set(minimumCapacity: minimumCapacity) - self.completion = completion - } - } - /// Applies the given edit to the current state and calls the editState's completion handler. - /// - Parameters: - /// - editState: The edit state to apply. - /// - startAtLayerIndex: An optional layer index to start from if some work has already been done on this edit - /// state object. - /// - runningAsync: Determine whether or not to timeout long running parse tasks. - internal func applyEdit(editState: EditState, startAtLayerIndex: Int? = nil, runningAsync: Bool = false) { - guard let readBlock, let textView, let state else { return } - stateLock.lock() + /// - Parameter edit: The edit to apply to the internal tree sitter state. + /// - Returns: The set of ranges invalidated by the edit operation. + func applyEdit(edit: InputEdit) -> IndexSet { + guard let state, let readBlock, let readCallback else { return IndexSet() } - // Loop through all layers, apply edits & find changed byte ranges. - let startIdx = startAtLayerIndex ?? 0 - for layerIdx in (startIdx..() + // Loop through all layers, apply edits & find changed byte ranges. + for (idx, layer) in state.layers.enumerated().reversed() { if layer.id != state.primaryLayer.id { // Reversed for safe removal while looping for rangeIdx in (0.. Void) - ) { - stateLock.lock() - guard let textView, let state = state?.copy() else { return } - stateLock.unlock() + func queryHighlightsForRange(range: NSRange) -> [HighlightRange] { + guard let state = self.state, let readCallback else { return [] } var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) @@ -26,12 +20,13 @@ extension TreeSitterClient { // Query injected only if a layer's ranges intersects with `range` for layerRange in layer.ranges { if let rangeIntersection = range.intersection(layerRange) { - highlights.append(contentsOf: queryLayerHighlights( + let queryResult = queryLayerHighlights( layer: layer, - textView: textView, - range: rangeIntersection - )) + range: rangeIntersection, + readCallback: readCallback + ) + highlights.append(contentsOf: queryResult) injectedSet.remove(integersIn: rangeIntersection) } } @@ -39,43 +34,26 @@ extension TreeSitterClient { // Query primary for any ranges that weren't used in the injected layers. for range in injectedSet.rangeView { - highlights.append(contentsOf: queryLayerHighlights( + let queryResult = queryLayerHighlights( layer: state.layers[0], - textView: textView, - range: NSRange(range) - )) + range: NSRange(range), + readCallback: readCallback + ) + highlights.append(contentsOf: queryResult) } - stateLock.unlock() - if !runningAsync { - completion(highlights) - } else { - DispatchQueue.main.async { - completion(highlights) - } - } - } - - internal func queryHighlightsForRangeAsync( - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void) - ) { - queuedQueries.append { - self.queryHighlightsForRange(range: range, runningAsync: true, completion: completion) - } - beginTasksIfNeeded() + return highlights } /// Queries the given language layer for any highlights. /// - Parameters: /// - layer: The layer to query. - /// - textView: A text view to use for contextual data. /// - range: The range to query for. /// - Returns: Any ranges to highlight. internal func queryLayerHighlights( layer: LanguageLayer, - textView: HighlighterTextView, - range: NSRange + range: NSRange, + readCallback: SwiftTreeSitter.Predicate.TextProvider ) -> [HighlightRange] { guard let tree = layer.tree, let rootNode = tree.rootNode else { @@ -84,13 +62,13 @@ extension TreeSitterClient { // This needs to be on the main thread since we're going to use the `textProvider` in // the `highlightsFromCursor` method, which uses the textView's text storage. - guard let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { + guard let queryCursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { return [] } - cursor.setRange(range) - cursor.matchLimit = Constants.treeSitterMatchLimit + queryCursor.setRange(range) + queryCursor.matchLimit = Constants.treeSitterMatchLimit - return highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor), includedRange: range) + return highlightsFromCursor(cursor: queryCursor, includedRange: range, readCallback: readCallback) } /// Resolves a query cursor to the highlight ranges it contains. @@ -99,15 +77,18 @@ extension TreeSitterClient { /// - cursor: The cursor to resolve. /// - includedRange: The range to include highlights from. /// - Returns: Any highlight ranges contained in the cursor. - internal func highlightsFromCursor(cursor: ResolvingQueryCursor, includedRange: NSRange) -> [HighlightRange] { - cursor.prepare(with: self.textProvider) + internal func highlightsFromCursor( + cursor: QueryCursor, + includedRange: NSRange, + readCallback: SwiftTreeSitter.Predicate.TextProvider + ) -> [HighlightRange] { return cursor .flatMap { $0.captures } .compactMap { - // Sometimes `cursor.setRange` just doesnt work :( so we have to do a redundant check for a valid range + // Sometimes `cursor.setRange` just doesn't work :( so we have to do a redundant check for a valid range // in the included range let intersectionRange = $0.range.intersection(includedRange) ?? .zero - // Check that the capture name is one CETV can parse. If not, ignore it completely. + // Check that the capture name is one CESE can parse. If not, ignore it completely. if intersectionRange.length > 0, let captureName = CaptureName.fromString($0.name ?? "") { return HighlightRange(range: intersectionRange, capture: captureName) } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift index 1f8c94098..88f83ac2f 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift @@ -6,49 +6,38 @@ // import Foundation +import CodeEditTextView import CodeEditLanguages import SwiftTreeSitter +import OSLog -/// `TreeSitterClient` is a class that manages applying edits for and querying captures for a syntax tree. -/// It handles queuing edits, processing them with the given text, and invalidating indices in the text for efficient -/// highlighting. +/// # TreeSitterClient /// -/// Use the `init` method to set up the client initially. If text changes it should be able to be read through the -/// `textProvider` callback. You can optionally update the text manually using the `setText` method. -/// However, the `setText` method will re-compile the entire corpus so should be used sparingly. -public final class TreeSitterClient: HighlightProviding { - typealias AsyncCallback = @Sendable () -> Void +/// ``TreeSitterClient`` is an actor type that manages a tree-sitter syntax tree and provides an API for notifying that +/// tree of edits and querying the tree. +/// +/// This type also conforms to ``HighlightProviding`` to provide syntax highlighting. +/// +public actor TreeSitterClient: HighlightProviding { + static let logger: Logger = Logger(subsystem: "com.CodeEdit.CodeEditSourceEditor", category: "TreeSitterClient") // MARK: - Properties - public var identifier: String { - "CodeEdit.TreeSitterClient" - } - - /// The text view to use as a data source for text. - internal weak var textView: HighlighterTextView? /// A callback to use to efficiently fetch portions of text. - internal var readBlock: Parser.ReadBlock? - - /// The running background task. - internal var runningTask: Task? - /// An array of all edits queued for execution. - internal var queuedEdits: [AsyncCallback] = [] - /// An array of all highlight queries queued for execution. - internal var queuedQueries: [AsyncCallback] = [] + var readBlock: Parser.ReadBlock? - /// A lock that must be obtained whenever `state` is modified - internal var stateLock: PthreadLock = PthreadLock() - /// A lock that must be obtained whenever either `queuedEdits` or `queuedHighlights` is modified - internal var queueLock: PthreadLock = PthreadLock() + /// A callback used to fetch text for queries. + var readCallback: SwiftTreeSitter.Predicate.TextProvider? /// The internal tree-sitter layer tree object. - internal var state: TreeSitterState? - internal var textProvider: ResolvingQueryCursor.TextProvider + var state: TreeSitterState? + + /// The end point of the previous edit. + private var oldEndPoint: Point? // MARK: - Constants - internal enum Constants { + enum Constants { /// The maximum amount of limits a cursor can match during a query. /// Used to ensure performance in large files, even though we generally limit the query to the visible range. /// Neovim encountered this issue and uses 64 for their limit. Helix uses 256 due to issues with some @@ -57,31 +46,9 @@ public final class TreeSitterClient: HighlightProviding { /// And: https://github.com/helix-editor/helix/pull/4830 static let treeSitterMatchLimit = 256 - /// The timeout for parsers. - static let parserTimeout: TimeInterval = 0.005 - - /// The maximum length of an edit before it must be processed asynchronously - static let maxSyncEditLength: Int = 1024 - - /// The maximum length a document can be before all queries and edits must be processed asynchronously. - static let maxSyncContentLength: Int = 1_000_000 - - /// The maximum length a query can be before it must be performed asynchronously. - static let maxSyncQueryLength: Int = 4096 - - /// The maximum number of highlight queries that can be performed in parallel. - static let simultaneousHighlightLimit: Int = 5 - } - - // MARK: - Init/Config - - /// Initializes the `TreeSitterClient` with the given parameters. - /// - Parameters: - /// - textView: The text view to use as a data source. - /// - codeLanguage: The language to set up the parser with. - /// - textProvider: The text provider callback to read any text. - public init(textProvider: @escaping ResolvingQueryCursor.TextProvider) { - self.textProvider = textProvider + /// The timeout for parsers to re-check if a task is canceled. This constant represents the period between + /// checks. + static let parserTimeout: TimeInterval = 0.1 } // MARK: - HighlightProviding @@ -91,46 +58,53 @@ public final class TreeSitterClient: HighlightProviding { /// - textView: The text view to use as a data source. /// A weak reference will be kept for the lifetime of this object. /// - codeLanguage: The language to use for parsing. - public func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) { - cancelAllRunningTasks() - queueLock.lock() - self.textView = textView - self.readBlock = textView.createReadBlock() - queuedEdits.append { - self.stateLock.lock() - self.state = TreeSitterState(codeLanguage: codeLanguage, textView: textView) - self.stateLock.unlock() + public func setUp(textView: TextView, codeLanguage: CodeLanguage) async { + self.readBlock = await textView.createReadBlock() + self.readCallback = await textView.createReadCallback() + + let task = Task.detached { + await self.setState( + newState: TreeSitterState( + codeLanguage: codeLanguage, + readCallback: self.readCallback!, + readBlock: self.readBlock! + ) + ) } - beginTasksIfNeeded() - queueLock.unlock() + await task.value + } + + private func setState(newState: TreeSitterState) async { + self.state = newState } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. /// - Parameters: - /// - textView:The text view to use. + /// - textView: The text view to use. /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - public func applyEdit( - textView: HighlighterTextView, - range: NSRange, - delta: Int, - completion: @escaping ((IndexSet) -> Void) - ) { - guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { return } - - queueLock.lock() - let longEdit = range.length > Constants.maxSyncEditLength - let longDocument = textView.documentRange.length > Constants.maxSyncContentLength - - if hasOutstandingWork || longEdit || longDocument { - applyEditAsync(editState: EditState(edit: edit, completion: completion), startAtLayerIndex: 0) - queueLock.unlock() + public func applyEdit(textView: TextView, range: NSRange, delta: Int) async -> IndexSet { + let oldEndPoint: Point + if self.oldEndPoint != nil { + oldEndPoint = self.oldEndPoint! } else { - queueLock.unlock() - applyEdit(editState: EditState(edit: edit, completion: completion)) + oldEndPoint = await textView.pointForLocation(range.max) ?? .zero + } + guard let edit = InputEdit( + range: range, + delta: delta, + oldEndPoint: oldEndPoint, + textView: textView + ) else { + return IndexSet() } + return applyEdit(edit: edit) + } + + public func willApplyEdit(textView: TextView, range: NSRange) async { + oldEndPoint = await textView.pointForLocation(range.max) } /// Initiates a highlight query. @@ -138,97 +112,7 @@ public final class TreeSitterClient: HighlightProviding { /// - textView: The text view to use. /// - range: The range to limit the highlights to. /// - completion: Called when the query completes. - public func queryHighlightsFor( - textView: HighlighterTextView, - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void) - ) { - queueLock.lock() - let longQuery = range.length > Constants.maxSyncQueryLength - let longDocument = textView.documentRange.length > Constants.maxSyncContentLength - - if hasOutstandingWork || longQuery || longDocument { - queryHighlightsForRangeAsync(range: range, completion: completion) - queueLock.unlock() - } else { - queueLock.unlock() - queryHighlightsForRange(range: range, runningAsync: false, completion: completion) - } - } - - // MARK: - Async - - /// Use to determine if there are any queued or running async tasks. - var hasOutstandingWork: Bool { - runningTask != nil || queuedEdits.count > 0 || queuedQueries.count > 0 - } - - private enum QueuedTaskType { - case edit(job: AsyncCallback) - case highlight(jobs: [AsyncCallback]) - } - - /// Spawn the running task if one is needed and doesn't already exist. - /// - /// The task will run until `determineNextTask` returns nil. It will run any highlight jobs in parallel. - internal func beginTasksIfNeeded() { - guard runningTask == nil && (queuedEdits.count > 0 || queuedQueries.count > 0) else { return } - runningTask = Task.detached(priority: .userInitiated) { - defer { - self.runningTask = nil - } - - do { - while let nextQueuedJob = self.determineNextJob() { - try Task.checkCancellation() - switch nextQueuedJob { - case .edit(let job): - job() - case .highlight(let jobs): - await withTaskGroup(of: Void.self, body: { taskGroup in - for job in jobs { - taskGroup.addTask { - job() - } - } - }) - } - } - } catch { } - } - } - - /// Determines the next async job to run and returns it if it exists. - /// Greedily returns queued highlight jobs determined by `Constants.simultaneousHighlightLimit` - private func determineNextJob() -> QueuedTaskType? { - queueLock.lock() - defer { - queueLock.unlock() - } - - // Get an edit task if any, otherwise get a highlight task if any. - if queuedEdits.count > 0 { - return .edit(job: queuedEdits.removeFirst()) - } else if queuedQueries.count > 0 { - let jobCount = min(queuedQueries.count, Constants.simultaneousHighlightLimit) - let jobs = Array(queuedQueries[0.. [HighlightRange] { + return queryHighlightsForRange(range: range) } } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift index 2aa24ae03..c77ece5e8 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift @@ -19,15 +19,19 @@ public class TreeSitterState { /// Initialize a state object with a language and text view. /// - Parameters: /// - codeLanguage: The language to use. - /// - textView: The text view to use as a text data source. - init(codeLanguage: CodeLanguage, textView: HighlighterTextView) { + /// - readCallback: Callback used to read text for a specific range. + /// - readBlock: Callback used to read blocks of text. + init( + codeLanguage: CodeLanguage, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock + ) { self.primaryLayer = codeLanguage self.setLanguage(codeLanguage) - let readBlock = textView.createReadBlock() layers[0].parser.timeout = 0.0 - layers[0].tree = layers[0].parser.createTree(readBlock: readBlock) + layers[0].tree = layers[0].parser.parse(tree: nil as Tree?, readBlock: readBlock) var layerSet = Set(arrayLiteral: layers[0]) var touchedLayers = Set() @@ -35,7 +39,8 @@ public class TreeSitterState { var idx = 0 while idx < layers.count { updateInjectedLanguageLayer( - textView: textView, + readCallback: readCallback, + readBlock: readBlock, layer: layers[idx], layerSet: &layerSet, touchedLayers: &touchedLayers @@ -45,7 +50,7 @@ public class TreeSitterState { } } - /// Private initilizer used by `copy` + /// Private initializer used by `copy` private init(codeLanguage: CodeLanguage, layers: [LanguageLayer]) { self.primaryLayer = codeLanguage self.layers = layers @@ -72,12 +77,6 @@ public class TreeSitterState { try? layers[0].parser.setLanguage(treeSitterLanguage) } - /// Creates a copy of this state object. - /// - Returns: The copied object - public func copy() -> TreeSitterState { - return TreeSitterState(codeLanguage: primaryLayer, layers: layers.map { $0.copy() }) - } - // MARK: - Layer Management /// Removes a layer at the given index. @@ -89,7 +88,7 @@ public class TreeSitterState { /// Removes all languagel ayers in the given set. /// - Parameter set: A set of all language layers to remove. public func removeLanguageLayers(in set: Set) { - layers.removeAll(where: { set.contains($0 )}) + layers.removeAll(where: { set.contains($0) }) } /// Attempts to create a language layer and load a highlights file. @@ -130,13 +129,15 @@ public class TreeSitterState { /// Inserts any new language layers, and removes any that may have been deleted after an edit. /// - Parameters: - /// - textView: The data source for text ranges. + /// - readCallback: Callback used to read text for a specific range. + /// - readBlock: Callback used to read blocks of text. /// - touchedLayers: A set of layers. Each time a layer is visited, it will be removed from this set. /// Use this to determine if any layers were not modified after this method was run. /// Those layers should be removed. /// - Returns: A set of indices of any new layers. This set indicates ranges that should be re-highlighted. public func updateInjectedLayers( - textView: HighlighterTextView, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock, touchedLayers: Set ) -> IndexSet { var layerSet = Set(layers) @@ -152,7 +153,8 @@ public class TreeSitterState { if layer.supportsInjections { rangeSet.formUnion( updateInjectedLanguageLayer( - textView: textView, + readCallback: readCallback, + readBlock: readBlock, layer: layer, layerSet: &layerSet, touchedLayers: &touchedLayers @@ -172,7 +174,8 @@ public class TreeSitterState { /// Performs an injections query on the given language layer. /// Updates any existing layers with new ranges and adds new layers if needed. /// - Parameters: - /// - textView: The text view to use. + /// - readCallback: Callback used to read text for a specific range. + /// - readBlock: Callback used to read blocks of text. /// - layer: The language layer to perform the query on. /// - layerSet: The set of layers that exist in the document. /// Used for efficient lookup of existing `(language, range)` pairs @@ -181,7 +184,8 @@ public class TreeSitterState { /// - Returns: An index set of any updated indexes. @discardableResult private func updateInjectedLanguageLayer( - textView: HighlighterTextView, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock, layer: LanguageLayer, layerSet: inout Set, touchedLayers: inout Set @@ -194,8 +198,8 @@ public class TreeSitterState { cursor.matchLimit = TreeSitterClient.Constants.treeSitterMatchLimit - let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in - return textView.stringForRange(range) + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, point in + return readCallback(range, point) } var updatedRanges = IndexSet() @@ -222,12 +226,11 @@ public class TreeSitterState { // If we've found this layer, it means it should exist after an edit. touchedLayers.remove(layer) } else { - let readBlock = textView.createReadBlock() // New range, make a new layer! if let addedLayer = addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) { addedLayer.ranges = [range.range] addedLayer.parser.includedRanges = addedLayer.ranges.map { $0.tsRange } - addedLayer.tree = addedLayer.parser.createTree(readBlock: readBlock) + addedLayer.tree = addedLayer.parser.parse(tree: nil as Tree?, readBlock: readBlock) layerSet.insert(addedLayer) updatedRanges.insert(range: range.range) @@ -247,7 +250,7 @@ public class TreeSitterState { /// - Returns: A map of each language to all the ranges they have been injected into. private func injectedLanguagesFrom( cursor: QueryCursor, - textProvider: @escaping ResolvingQueryCursor.TextProvider + textProvider: @escaping SwiftTreeSitter.Predicate.TextProvider ) -> [String: [NamedRange]] { var languages: [String: [NamedRange]] = [:] diff --git a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift index 13467e8a0..1a5d02bb8 100644 --- a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift @@ -14,96 +14,94 @@ fileprivate class TestTextView: HighlighterTextView { } } -final class TreeSitterClientTests: XCTestCase { - - fileprivate var textView = TestTextView() - var client: TreeSitterClient! - - override func setUp() { - client = TreeSitterClient { nsRange, _ in - self.textView.stringForRange(nsRange) - } - } - - func test_clientSetup() { - client.setUp(textView: textView, codeLanguage: .swift) - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - - let editExpectation = expectation(description: "Edit work should never return") - editExpectation.isInverted = true // Expect to never happen - - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - editExpectation.fulfill() - } - - client.setUp(textView: textView, codeLanguage: .swift) - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - XCTAssert(client.queuedEdits.count == 1, "Client should cancel all queued work when setUp is called.") - - waitForExpectations(timeout: 1.0, handler: nil) - } - - // Test async language loading with edits and highlights queued before loading completes. - func test_languageLoad() { - textView = TestTextView() - client.setUp(textView: textView, codeLanguage: .swift) - - XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") - - let editExpectation = expectation(description: "Edit work should return first.") - let highlightExpectation = expectation(description: "Highlight should return last.") - - client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in - highlightExpectation.fulfill() - } - - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - editExpectation.fulfill() - } - - wait(for: [editExpectation, highlightExpectation], timeout: 10.0, enforceOrder: true) - } - - // Edits should be consumed before highlights. - func test_queueOrder() { - textView = TestTextView() - client.setUp(textView: textView, codeLanguage: .swift) - - let editExpectation = expectation(description: "Edit work should return first.") - let editExpectation2 = expectation(description: "Edit2 should return 2nd.") - let highlightExpectation = expectation(description: "Highlight should return 3rd.") - - // Do initial query while language loads. - client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in - print("highlightExpectation") - highlightExpectation.fulfill() - } - - // Queue another edit - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - print("editExpectation") - editExpectation.fulfill() - } - - // One more edit - textView.testString.insert("let int = 0\n", at: 0) - client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in - print("editExpectation2") - editExpectation2.fulfill() - } - - wait( - for: [ - editExpectation, - editExpectation2, - highlightExpectation, - ], - timeout: 10.0, - enforceOrder: true - ) - } -} +//final class TreeSitterClientTests: XCTestCase { +// +// fileprivate var textView = TestTextView() +// var client: TreeSitterClient! +// +// override func setUp() { +// client = TreeSitterClient { nsRange, _ in +// self.textView.stringForRange(nsRange) +// } +// } +// +// func test_clientSetup() async { +// client.setUp(textView: textView, codeLanguage: .swift) +// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") +// +// let editExpectation = expectation(description: "Edit work should never return") +// editExpectation.isInverted = true // Expect to never happen +// +// textView.testString.insert("let int = 0\n", at: 0) +// let _ = await client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) +// +// await client.setUp(textView: textView, codeLanguage: .swift) +// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") +// XCTAssert(client.queuedEdits.count == 1, "Client should cancel all queued work when setUp is called.") +// +// waitForExpectations(timeout: 1.0, handler: nil) +// } +// +// // Test async language loading with edits and highlights queued before loading completes. +// func test_languageLoad() { +// textView = TestTextView() +// client.setUp(textView: textView, codeLanguage: .swift) +// +// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") +// +// let editExpectation = expectation(description: "Edit work should return first.") +// let highlightExpectation = expectation(description: "Highlight should return last.") +// +// client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in +// highlightExpectation.fulfill() +// } +// +// textView.testString.insert("let int = 0\n", at: 0) +// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in +// editExpectation.fulfill() +// } +// +// wait(for: [editExpectation, highlightExpectation], timeout: 10.0, enforceOrder: true) +// } +// +// // Edits should be consumed before highlights. +// func test_queueOrder() { +// textView = TestTextView() +// client.setUp(textView: textView, codeLanguage: .swift) +// +// let editExpectation = expectation(description: "Edit work should return first.") +// let editExpectation2 = expectation(description: "Edit2 should return 2nd.") +// let highlightExpectation = expectation(description: "Highlight should return 3rd.") +// +// // Do initial query while language loads. +// client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in +// print("highlightExpectation") +// highlightExpectation.fulfill() +// } +// +// // Queue another edit +// textView.testString.insert("let int = 0\n", at: 0) +// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in +// print("editExpectation") +// editExpectation.fulfill() +// } +// +// // One more edit +// textView.testString.insert("let int = 0\n", at: 0) +// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in +// print("editExpectation2") +// editExpectation2.fulfill() +// } +// +// wait( +// for: [ +// editExpectation, +// editExpectation2, +// highlightExpectation, +// ], +// timeout: 10.0, +// enforceOrder: true +// ) +// } +//} // swiftlint:enable all From 4ba027b3489981ef7213794e09e9b37e5e2dac85 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sun, 21 Jan 2024 17:58:22 -0600 Subject: [PATCH 2/9] Make Highlighter Track & Cancel Tasks --- Package.resolved | 11 +++- Package.swift | 2 +- .../Highlighter+NSTextStorageDelegate.swift | 10 ++-- .../Highlighting/Highlighter.swift | 52 ++++++++++++------- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/Package.resolved b/Package.resolved index de9cb3ae1..f9a1ff7af 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,14 @@ { "pins" : [ + { + "identity" : "codeedittextview", + "kind" : "remoteSourceControl", + "location" : "https://github.com/CodeEditApp/CodeEditTextView.git", + "state" : { + "revision" : "6abce20f1827a3665a5159195157f592352e38b4", + "version" : "0.7.1" + } + }, { "identity" : "mainoffender", "kind" : "remoteSourceControl", @@ -39,7 +48,7 @@ { "identity" : "swifttreesitter", "kind" : "remoteSourceControl", - "location" : "https://github.com/ChimeHQ/SwiftTreeSitter", + "location" : "https://github.com/ChimeHQ/SwiftTreeSitter.git", "state" : { "revision" : "2599e95310b3159641469d8a21baf2d3d200e61f", "version" : "0.8.0" diff --git a/Package.swift b/Package.swift index 8d8837275..2dfe6a02a 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,7 @@ let package = Package( // tree-sitter languages .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - exact: "0.1.17" + exact: "0.1.18" ), // SwiftLint .package( diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift index 0edd3fcd5..1cd86be4d 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift @@ -20,11 +20,10 @@ extension Highlighter: NSTextStorageDelegate { // each time an attribute is applied, we check to make sure this is in response to an edit. guard editedMask.contains(.editedCharacters) else { return } - let editTask = Task { - await storageDidEdit(editedRange: editedRange, delta: delta) + addTask { + await self.storageDidEdit(editedRange: editedRange, delta: delta) return // To keep the compiler happy } - runningTasks.insert(editTask) } func textStorage( @@ -35,10 +34,9 @@ extension Highlighter: NSTextStorageDelegate { ) { guard editedMask.contains(.editedCharacters) else { return } - let editTask = Task { - await storageWillEdit(editedRange: editedRange) + addTask { + await self.storageWillEdit(editedRange: editedRange) return; } - runningTasks.insert(editTask) } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift index eb3c26ed9..a301e49fc 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift @@ -38,7 +38,7 @@ class Highlighter: NSObject { // MARK: - Tasks - var runningTasks: Set> = [] + private var runningTasks: [UUID: Task] = [:] // MARK: - UI @@ -82,11 +82,10 @@ class Highlighter: NSObject { super.init() - let setupTask = Task { + addTask { await highlightProvider?.setUp(textView: textView, codeLanguage: language) return } - runningTasks.insert(setupTask) if let scrollView = textView.enclosingScrollView { NotificationCenter.default.addObserver( @@ -119,13 +118,12 @@ class Highlighter: NSObject { public func setLanguage(language: CodeLanguage) { cancelAllTasks() - let setLanguageTask = Task { - guard let textView else { return } - await highlightProvider?.setUp(textView: textView, codeLanguage: language) + addTask { + guard let textView = self.textView else { return } + await self.highlightProvider?.setUp(textView: textView, codeLanguage: language) guard !Task.isCancelled else { return } - invalidate() + self.invalidate() } - runningTasks.insert(setLanguageTask) } /// Sets the highlight provider. Will cause a re-highlight of the entire text. @@ -134,25 +132,41 @@ class Highlighter: NSObject { cancelAllTasks() highlightProvider = provider - let setHighlightProviderTask = Task { - guard let textView else { return } - await highlightProvider?.setUp(textView: textView, codeLanguage: language) + addTask { + guard let textView = self.textView else { return } + await self.highlightProvider?.setUp(textView: textView, codeLanguage: self.language) guard !Task.isCancelled else { return } - invalidate() + self.invalidate() } - runningTasks.insert(setHighlightProviderTask) + } + + /// Add a task to the set of tracked tasks for this highlighter. + /// + /// This method wraps the operation in a task that will remove itself from the list of running tasks, allowing + /// this class to track and cancel tasks itself. + /// + /// - Parameters: + /// - detached: Set to true to detach the task from the current context. + /// - operation: The operation to perform asynchronously. + func addTask(detached: Bool = false, operation: @MainActor @Sendable @escaping () async -> Void) { + // Add the new task to the running tasks list. + let taskId = UUID() + let newTask = Task { + await operation() + runningTasks.removeValue(forKey: taskId) + } + runningTasks[taskId] = newTask } func cancelAllTasks() { - for task in runningTasks { + for task in runningTasks.values { task.cancel() } runningTasks.removeAll() } deinit { - print("Highlighter deinit") - for task in runningTasks { + for task in runningTasks.values { task.cancel() } runningTasks.removeAll() @@ -199,8 +213,7 @@ private extension Highlighter { for range in rangesToHighlight { pendingSet.insert(integersIn: range) } - - let queryTask = Task.detached { + addTask(detached: true) { await withTaskGroup(of: Void.self) { group in for range in rangesToHighlight { group.addTask { [weak self] in @@ -210,7 +223,7 @@ private extension Highlighter { textView: textView, range: range ) - + guard !Task.isCancelled else { return } await self?.applyHighlightResult(highlights ?? [], rangeToHighlight: range) @@ -218,7 +231,6 @@ private extension Highlighter { } } } - runningTasks.insert(queryTask) } /// Applies a highlight query result to the text view. From 3451672ba4d706ebd9a598356fdd4e5089d79d2c Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:59:14 -0600 Subject: [PATCH 3/9] Lint Fixes --- .../CodeEditSourceEditor/Highlighting/HighlightProviding.swift | 2 +- .../Highlighting/Highlighter+NSTextStorageDelegate.swift | 2 +- Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift | 2 +- Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift index 56ed660af..6452f6ab1 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift @@ -17,7 +17,7 @@ public protocol HighlightProviding: AnyObject { /// - textView: The text view to use as a text source. /// - codeLanguage: The language that should be used by the highlighter. func setUp(textView: TextView, codeLanguage: CodeLanguage) async - + /// Notifies the highlighter that an edit is going to happen in the given range. /// - Parameters: /// - textView: The text view to use. diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift index 1cd86be4d..f9b8016c0 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift @@ -36,7 +36,7 @@ extension Highlighter: NSTextStorageDelegate { addTask { await self.storageWillEdit(editedRange: editedRange) - return; + return } } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift index a301e49fc..9c61edb17 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift @@ -139,7 +139,7 @@ class Highlighter: NSObject { self.invalidate() } } - + /// Add a task to the set of tracked tasks for this highlighter. /// /// This method wraps the operation in a task that will remove itself from the list of running tasks, allowing diff --git a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift index 39205b8d4..8421f1c6e 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift @@ -122,7 +122,7 @@ public class LanguageLayer: Hashable { // Check every timeout to see if the task is canceled to avoid parsing after the editor has been closed. // We can continue a parse after a timeout causes it to cancel by calling parse on the same tree. - var newTree: MutableTree? = nil + var newTree: MutableTree? while newTree == nil && !Task.isCancelled { newTree = parser.parse(tree: tree, readBlock: readBlock) } From b376e5d9a8fef81efff32610f0f7f8aedb98b080 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:07:45 -0600 Subject: [PATCH 4/9] Update Tests --- Package.resolved | 9 ++ .../TreeSitterClientTests.swift | 124 ++++-------------- 2 files changed, 36 insertions(+), 97 deletions(-) diff --git a/Package.resolved b/Package.resolved index f9a1ff7af..5d97a3e82 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,14 @@ { "pins" : [ + { + "identity" : "codeeditlanguages", + "kind" : "remoteSourceControl", + "location" : "https://github.com/CodeEditApp/CodeEditLanguages.git", + "state" : { + "revision" : "620b463c88894741e20d4711c9435b33547de5d2", + "version" : "0.1.18" + } + }, { "identity" : "codeedittextview", "kind" : "remoteSourceControl", diff --git a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift index 1a5d02bb8..f61862abd 100644 --- a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift @@ -1,107 +1,37 @@ import XCTest +import CodeEditTextView @testable import CodeEditSourceEditor // swiftlint:disable all -fileprivate class TestTextView: HighlighterTextView { - var testString: NSMutableString = "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}" - var documentRange: NSRange { - NSRange(location: 0, length: testString.length) +final class TreeSitterClientTests: XCTestCase { + + class Delegate: TextViewDelegate { } + + fileprivate var textView = TextView( + string: "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}", + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: true, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: Delegate() + ) + var client: TreeSitterClient! + + override func setUp() { + client = TreeSitterClient() } - func stringForRange(_ nsRange: NSRange) -> String? { - testString.substring(with: nsRange) + func test_clientSetup() async { + await client.setUp(textView: textView, codeLanguage: .swift) + + let primaryLanguage = await client.state?.primaryLayer.id + let layerCount = await client.state?.layers.count + XCTAssert(primaryLanguage == .swift, "Client set up incorrect language") + XCTAssert(layerCount == 1, "Client set up too many layers") } } - -//final class TreeSitterClientTests: XCTestCase { -// -// fileprivate var textView = TestTextView() -// var client: TreeSitterClient! -// -// override func setUp() { -// client = TreeSitterClient { nsRange, _ in -// self.textView.stringForRange(nsRange) -// } -// } -// -// func test_clientSetup() async { -// client.setUp(textView: textView, codeLanguage: .swift) -// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") -// -// let editExpectation = expectation(description: "Edit work should never return") -// editExpectation.isInverted = true // Expect to never happen -// -// textView.testString.insert("let int = 0\n", at: 0) -// let _ = await client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) -// -// await client.setUp(textView: textView, codeLanguage: .swift) -// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") -// XCTAssert(client.queuedEdits.count == 1, "Client should cancel all queued work when setUp is called.") -// -// waitForExpectations(timeout: 1.0, handler: nil) -// } -// -// // Test async language loading with edits and highlights queued before loading completes. -// func test_languageLoad() { -// textView = TestTextView() -// client.setUp(textView: textView, codeLanguage: .swift) -// -// XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") -// -// let editExpectation = expectation(description: "Edit work should return first.") -// let highlightExpectation = expectation(description: "Highlight should return last.") -// -// client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in -// highlightExpectation.fulfill() -// } -// -// textView.testString.insert("let int = 0\n", at: 0) -// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in -// editExpectation.fulfill() -// } -// -// wait(for: [editExpectation, highlightExpectation], timeout: 10.0, enforceOrder: true) -// } -// -// // Edits should be consumed before highlights. -// func test_queueOrder() { -// textView = TestTextView() -// client.setUp(textView: textView, codeLanguage: .swift) -// -// let editExpectation = expectation(description: "Edit work should return first.") -// let editExpectation2 = expectation(description: "Edit2 should return 2nd.") -// let highlightExpectation = expectation(description: "Highlight should return 3rd.") -// -// // Do initial query while language loads. -// client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in -// print("highlightExpectation") -// highlightExpectation.fulfill() -// } -// -// // Queue another edit -// textView.testString.insert("let int = 0\n", at: 0) -// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in -// print("editExpectation") -// editExpectation.fulfill() -// } -// -// // One more edit -// textView.testString.insert("let int = 0\n", at: 0) -// client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in -// print("editExpectation2") -// editExpectation2.fulfill() -// } -// -// wait( -// for: [ -// editExpectation, -// editExpectation2, -// highlightExpectation, -// ], -// timeout: 10.0, -// enforceOrder: true -// ) -// } -//} // swiftlint:enable all From bfffd234653e3cd8b09153fa81d929b7fc45d746 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:10:41 -0600 Subject: [PATCH 5/9] Fix failing test --- .../TextViewController+HighlightBracket.swift | 6 +-- .../TreeSitter/TreeSitterClient.swift | 17 ++++--- .../TreeSitter/TreeSitterState.swift | 49 +++++++++++-------- .../TextViewControllerTests.swift | 2 +- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift index 86851f080..0622d5e92 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController+HighlightBracket.swift @@ -15,11 +15,11 @@ extension TextViewController { for range in textView.selectionManager.textSelections.map({ $0.range }) { if range.isEmpty, range.location > 0, // Range is not the beginning of the document - let preceedingCharacter = textView.textStorage.substring( + let precedingCharacter = textView.textStorage.substring( from: NSRange(location: range.location - 1, length: 1) // The preceding character exists ) { for pair in BracketPairs.allValues { - if preceedingCharacter == pair.0 { + if precedingCharacter == pair.0 { // Walk forwards if let characterIndex = findClosingPair( pair.0, @@ -34,7 +34,7 @@ extension TextViewController { highlightCharacter(range.location - 1) } } - } else if preceedingCharacter == pair.1 && range.location - 1 > 0 { + } else if precedingCharacter == pair.1 && range.location - 1 > 0 { // Walk backwards if let characterIndex = findClosingPair( pair.1, diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift index 88f83ac2f..69217942e 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift @@ -64,18 +64,21 @@ public actor TreeSitterClient: HighlightProviding { let task = Task.detached { await self.setState( - newState: TreeSitterState( - codeLanguage: codeLanguage, - readCallback: self.readCallback!, - readBlock: self.readBlock! - ) + language: codeLanguage, + readCallback: self.readCallback!, + readBlock: self.readBlock! ) } await task.value } - private func setState(newState: TreeSitterState) async { - self.state = newState + private func setState( + language: CodeLanguage, + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock + ) async { + self.state?.setLanguage(language) + self.state?.parseDocument(readCallback: readCallback, readBlock: readBlock) } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift index c77ece5e8..d8dde99c8 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift @@ -27,27 +27,8 @@ public class TreeSitterState { readBlock: @escaping Parser.ReadBlock ) { self.primaryLayer = codeLanguage - self.setLanguage(codeLanguage) - - layers[0].parser.timeout = 0.0 - layers[0].tree = layers[0].parser.parse(tree: nil as Tree?, readBlock: readBlock) - - var layerSet = Set(arrayLiteral: layers[0]) - var touchedLayers = Set() - - var idx = 0 - while idx < layers.count { - updateInjectedLanguageLayer( - readCallback: readCallback, - readBlock: readBlock, - layer: layers[idx], - layerSet: &layerSet, - touchedLayers: &touchedLayers - ) - - idx += 1 - } + self.parseDocument(readCallback: readCallback, readBlock: readBlock) } /// Private initializer used by `copy` @@ -77,6 +58,34 @@ public class TreeSitterState { try? layers[0].parser.setLanguage(treeSitterLanguage) } + /// Performs the initial document parse for the primary layer. + /// - Parameters: + /// - readCallback: The callback to use to read content from the document. + /// - readBlock: The callback to use to read blocks of content from the document. + public func parseDocument( + readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, + readBlock: @escaping Parser.ReadBlock + ) { + layers[0].parser.timeout = 0.0 + layers[0].tree = layers[0].parser.parse(tree: nil as Tree?, readBlock: readBlock) + + var layerSet = Set(arrayLiteral: layers[0]) + var touchedLayers = Set() + + var idx = 0 + while idx < layers.count { + updateInjectedLanguageLayer( + readCallback: readCallback, + readBlock: readBlock, + layer: layers[idx], + layerSet: &layerSet, + touchedLayers: &touchedLayers + ) + + idx += 1 + } + } + // MARK: - Layer Management /// Removes a layer at the given index. diff --git a/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift b/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift index 9c84bdb40..fc5e340fa 100644 --- a/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift +++ b/Tests/CodeEditSourceEditorTests/TextViewControllerTests.swift @@ -237,7 +237,7 @@ final class TextViewControllerTests: XCTestCase { controller.scrollView.setFrameSize(NSSize(width: 500, height: 500)) controller.viewDidLoad() controller.bracketPairHighlight = nil - controller.textView.string = "{ Loren Ipsum {} }" + controller.setText("{ Loren Ipsum {} }") controller.setCursorPositions([CursorPosition(line: 1, column: 2)]) // After first opening { XCTAssert(controller.highlightLayers.isEmpty, "Controller added highlight layer when setting is set to `nil`") controller.setCursorPositions([CursorPosition(line: 1, column: 3)]) From 7f44a92814c28985023202fe8f7eaf1a3c9c7682 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:17:46 -0600 Subject: [PATCH 6/9] Fixed tree sitter init --- .../TreeSitter/TreeSitterClient.swift | 8 ++++++-- .../CodeEditSourceEditor/TreeSitter/TreeSitterState.swift | 4 ++-- .../CodeEditSourceEditorTests/TreeSitterClientTests.swift | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift index 69217942e..15f26d710 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift @@ -72,13 +72,17 @@ public actor TreeSitterClient: HighlightProviding { await task.value } + /// Sets the client's new state. + /// - Parameters: + /// - language: The language to use. + /// - readCallback: The callback to use to read text from the document. + /// - readBlock: The callback to use to read blocks of text from the document. private func setState( language: CodeLanguage, readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, readBlock: @escaping Parser.ReadBlock ) async { - self.state?.setLanguage(language) - self.state?.parseDocument(readCallback: readCallback, readBlock: readBlock) + self.state = TreeSitterState(codeLanguage: language, readCallback: readCallback, readBlock: readBlock) } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift index d8dde99c8..0508960e2 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterState.swift @@ -39,7 +39,7 @@ public class TreeSitterState { /// Sets the language for the state. Removing all existing layers. /// - Parameter codeLanguage: The language to use. - public func setLanguage(_ codeLanguage: CodeLanguage) { + private func setLanguage(_ codeLanguage: CodeLanguage) { layers.removeAll() primaryLayer = codeLanguage @@ -62,7 +62,7 @@ public class TreeSitterState { /// - Parameters: /// - readCallback: The callback to use to read content from the document. /// - readBlock: The callback to use to read blocks of content from the document. - public func parseDocument( + private func parseDocument( readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, readBlock: @escaping Parser.ReadBlock ) { diff --git a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift index f61862abd..c7bf6e0ab 100644 --- a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift @@ -30,8 +30,8 @@ final class TreeSitterClientTests: XCTestCase { let primaryLanguage = await client.state?.primaryLayer.id let layerCount = await client.state?.layers.count - XCTAssert(primaryLanguage == .swift, "Client set up incorrect language") - XCTAssert(layerCount == 1, "Client set up too many layers") + XCTAssertEqual(primaryLanguage, .swift, "Client set up incorrect language") + XCTAssertEqual(layerCount, 1, "Client set up too many layers") } } // swiftlint:enable all From 9d32e53f626aacadebd1a279c7a8de88aea2feb2 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:12:48 -0600 Subject: [PATCH 7/9] Make self weak in textview callback --- .../TextView+createReadBlock.swift} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename Sources/CodeEditSourceEditor/Extensions/{HighlighterTextView+createReadBlock.swift => TextView+/TextView+createReadBlock.swift} (91%) diff --git a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift similarity index 91% rename from Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift rename to Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift index c0d58fc80..89eb4d16a 100644 --- a/Sources/CodeEditSourceEditor/Extensions/HighlighterTextView+createReadBlock.swift +++ b/Sources/CodeEditSourceEditor/Extensions/TextView+/TextView+createReadBlock.swift @@ -25,8 +25,8 @@ extension TextView { } func createReadCallback() -> SwiftTreeSitter.Predicate.TextProvider { - return { range, _ in - return self.stringForRange(range) + return { [weak self] range, _ in + return self?.stringForRange(range) } } } From e79257c10f42448f5b2d0e4b8ffc87d9a451a463 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 23 Jan 2024 13:55:22 -0600 Subject: [PATCH 8/9] Fix Typo --- .../Extensions/NSRange+/NSRange+InputEdit.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift index 1ef871771..3bc073686 100644 --- a/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditSourceEditor/Extensions/NSRange+/NSRange+InputEdit.swift @@ -19,7 +19,7 @@ extension InputEdit { } let newRange = NSRange(location: range.location, length: range.length + delta) - let startPoint = textView.pointForLocation(range.location) ?? .zero + let startPoint = textView.pointForLocation(newRange.location) ?? .zero let newEndPoint = textView.pointForLocation(newEndLocation) ?? .zero self.init( From b948af7dbd9ef6d19ed8cff2c5b8376b18a012fa Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:38:34 -0600 Subject: [PATCH 9/9] Switch to `DispatchQueue` for async --- .../TextViewController+Highlighter.swift | 1 - .../Controller/TextViewController.swift | 5 - .../Highlighting/HighlightProviding.swift | 10 +- .../Highlighter+NSTextStorageDelegate.swift | 10 +- .../Highlighting/Highlighter.swift | 137 +++++-------- .../TreeSitter/LanguageLayer.swift | 9 +- .../TreeSitter/TreeSitterClient.swift | 189 +++++++++++++++--- .../TreeSitterClientTests.swift | 19 +- 8 files changed, 233 insertions(+), 147 deletions(-) diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift index c64a49b7f..0ad8597e9 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController+Highlighter.swift @@ -12,7 +12,6 @@ extension TextViewController { internal func setUpHighlighter() { if let highlighter { textView.removeStorageDelegate(highlighter) - highlighter.cancelAllTasks() self.highlighter = nil } diff --git a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift index ee74a1710..2d5dc4b4d 100644 --- a/Sources/CodeEditSourceEditor/Controller/TextViewController.swift +++ b/Sources/CodeEditSourceEditor/Controller/TextViewController.swift @@ -297,11 +297,6 @@ public class TextViewController: NSViewController { deinit { if let highlighter { textView.removeStorageDelegate(highlighter) - Task { - // We can safely do this async operation here b/c the highlighter will not deinit until its - // tasks are finished. - await highlighter.cancelAllTasks() - } } highlighter = nil highlightProvider = nil diff --git a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift index 6452f6ab1..3d23beb8a 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/HighlightProviding.swift @@ -16,13 +16,13 @@ public protocol HighlightProviding: AnyObject { /// - Parameters: /// - textView: The text view to use as a text source. /// - codeLanguage: The language that should be used by the highlighter. - func setUp(textView: TextView, codeLanguage: CodeLanguage) async + func setUp(textView: TextView, codeLanguage: CodeLanguage) /// Notifies the highlighter that an edit is going to happen in the given range. /// - Parameters: /// - textView: The text view to use. /// - range: The range of the incoming edit. - func willApplyEdit(textView: TextView, range: NSRange) async + func willApplyEdit(textView: TextView, range: NSRange) /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. @@ -31,7 +31,7 @@ public protocol HighlightProviding: AnyObject { /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - Returns: an `IndexSet` containing all Indices to invalidate. - func applyEdit(textView: TextView, range: NSRange, delta: Int) async -> IndexSet + func applyEdit(textView: TextView, range: NSRange, delta: Int, completion: @escaping (IndexSet) -> Void) /// Queries the highlight provider for any ranges to apply highlights to. The highlight provider should return an /// array containing all ranges to highlight, and the capture type for the range. Any ranges or indexes @@ -40,9 +40,9 @@ public protocol HighlightProviding: AnyObject { /// - textView: The text view to use. /// - range: The range to query. /// - Returns: All highlight ranges for the queried ranges. - func queryHighlightsFor(textView: TextView, range: NSRange) async -> [HighlightRange] + func queryHighlightsFor(textView: TextView, range: NSRange, completion: @escaping ([HighlightRange]) -> Void) } extension HighlightProviding { - public func willApplyEdit(textView: TextView, range: NSRange) async { } + public func willApplyEdit(textView: TextView, range: NSRange) { } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift index f9b8016c0..096302641 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter+NSTextStorageDelegate.swift @@ -20,10 +20,7 @@ extension Highlighter: NSTextStorageDelegate { // each time an attribute is applied, we check to make sure this is in response to an edit. guard editedMask.contains(.editedCharacters) else { return } - addTask { - await self.storageDidEdit(editedRange: editedRange, delta: delta) - return // To keep the compiler happy - } + self.storageDidEdit(editedRange: editedRange, delta: delta) } func textStorage( @@ -34,9 +31,6 @@ extension Highlighter: NSTextStorageDelegate { ) { guard editedMask.contains(.editedCharacters) else { return } - addTask { - await self.storageWillEdit(editedRange: editedRange) - return - } + self.storageWillEdit(editedRange: editedRange) } } diff --git a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift index 9c61edb17..8beebf067 100644 --- a/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift +++ b/Sources/CodeEditSourceEditor/Highlighting/Highlighter.swift @@ -58,7 +58,7 @@ class Highlighter: NSObject { private(set) weak var highlightProvider: HighlightProviding? /// The length to chunk ranges into when passing to the highlighter. - private let rangeChunkLimit = 256 + private let rangeChunkLimit = 1024 // MARK: - Init @@ -82,10 +82,7 @@ class Highlighter: NSObject { super.init() - addTask { - await highlightProvider?.setUp(textView: textView, codeLanguage: language) - return - } + highlightProvider?.setUp(textView: textView, codeLanguage: language) if let scrollView = textView.enclosingScrollView { NotificationCenter.default.addObserver( @@ -116,60 +113,21 @@ class Highlighter: NSObject { /// Sets the language and causes a re-highlight of the entire text. /// - Parameter language: The language to update to. public func setLanguage(language: CodeLanguage) { - cancelAllTasks() - - addTask { - guard let textView = self.textView else { return } - await self.highlightProvider?.setUp(textView: textView, codeLanguage: language) - guard !Task.isCancelled else { return } - self.invalidate() - } + guard let textView = self.textView else { return } + highlightProvider?.setUp(textView: textView, codeLanguage: language) + invalidate() } /// Sets the highlight provider. Will cause a re-highlight of the entire text. /// - Parameter provider: The provider to use for future syntax highlights. public func setHighlightProvider(_ provider: HighlightProviding) { - cancelAllTasks() - - highlightProvider = provider - addTask { - guard let textView = self.textView else { return } - await self.highlightProvider?.setUp(textView: textView, codeLanguage: self.language) - guard !Task.isCancelled else { return } - self.invalidate() - } - } - - /// Add a task to the set of tracked tasks for this highlighter. - /// - /// This method wraps the operation in a task that will remove itself from the list of running tasks, allowing - /// this class to track and cancel tasks itself. - /// - /// - Parameters: - /// - detached: Set to true to detach the task from the current context. - /// - operation: The operation to perform asynchronously. - func addTask(detached: Bool = false, operation: @MainActor @Sendable @escaping () async -> Void) { - // Add the new task to the running tasks list. - let taskId = UUID() - let newTask = Task { - await operation() - runningTasks.removeValue(forKey: taskId) - } - runningTasks[taskId] = newTask - } - - func cancelAllTasks() { - for task in runningTasks.values { - task.cancel() - } - runningTasks.removeAll() + self.highlightProvider = provider + guard let textView = self.textView else { return } + highlightProvider?.setUp(textView: textView, codeLanguage: self.language) + invalidate() } deinit { - for task in runningTasks.values { - task.cancel() - } - runningTasks.removeAll() self.attributeProvider = nil self.textView = nil self.highlightProvider = nil @@ -210,26 +168,25 @@ private extension Highlighter { /// Highlights the given ranges /// - Parameter ranges: The ranges to request highlights for. func queryHighlights(for rangesToHighlight: [NSRange]) { - for range in rangesToHighlight { - pendingSet.insert(integersIn: range) - } - addTask(detached: true) { - await withTaskGroup(of: Void.self) { group in - for range in rangesToHighlight { - group.addTask { [weak self] in - guard let textView = await self?.textView else { return } - - let highlights = await self?.highlightProvider?.queryHighlightsFor( - textView: textView, - range: range - ) - - guard !Task.isCancelled else { return } + guard let textView else { return } - await self?.applyHighlightResult(highlights ?? [], rangeToHighlight: range) + if !Thread.isMainThread { + DispatchQueue.main.async { [weak self] in + for range in rangesToHighlight { + self?.highlightProvider?.queryHighlightsFor( + textView: textView, + range: range + ) { [weak self] highlights in + self?.applyHighlightResult(highlights, rangeToHighlight: range) } } } + } else { + for range in rangesToHighlight { + highlightProvider?.queryHighlightsFor(textView: textView, range: range) { [weak self] highlights in + self?.applyHighlightResult(highlights, rangeToHighlight: range) + } + } } } @@ -237,7 +194,6 @@ private extension Highlighter { /// - Parameters: /// - results: The result of a highlight query. /// - rangeToHighlight: The range to apply the highlight to. - @MainActor private func applyHighlightResult(_ results: [HighlightRange], rangeToHighlight: NSRange) { guard let attributeProvider = self.attributeProvider else { return @@ -257,7 +213,8 @@ private extension Highlighter { var ignoredIndexes = IndexSet(integersIn: rangeToHighlight) // Apply all highlights that need color - for highlight in results { + for highlight in results + where textView?.documentRange.upperBound ?? 0 > highlight.range.upperBound { textView?.textStorage.setAttributes( attributeProvider.attributesFor(highlight.capture), range: highlight.range @@ -270,7 +227,8 @@ private extension Highlighter { // For any indices left over, we need to apply normal attributes to them // This fixes the case where characters are changed to have a non-text color, and then are skipped when // they need to be changed back. - for ignoredRange in ignoredIndexes.rangeView { + for ignoredRange in ignoredIndexes.rangeView + where textView?.documentRange.upperBound ?? 0 > ignoredRange.upperBound { textView?.textStorage.setAttributes( attributeProvider.attributesFor(nil), range: NSRange(ignoredRange) @@ -313,10 +271,17 @@ private extension Highlighter { /// Updates the view to highlight newly visible text when the textview is scrolled or bounds change. @objc func visibleTextChanged(_ notification: Notification) { - guard let clipView = notification.object as? NSClipView, - let textView = clipView.enclosingScrollView?.documentView as? TextView else { + let textView: TextView + if let clipView = notification.object as? NSClipView, + let documentView = clipView.enclosingScrollView?.documentView as? TextView { + textView = documentView + } else if let scrollView = notification.object as? NSScrollView, + let documentView = scrollView.documentView as? TextView { + textView = documentView + } else { return } + updateVisibleSet(textView: textView) // Any indices that are both *not* valid and in the visible text range should be invalidated @@ -331,7 +296,7 @@ private extension Highlighter { // MARK: - Editing extension Highlighter { - func storageDidEdit(editedRange: NSRange, delta: Int) async { + func storageDidEdit(editedRange: NSRange, delta: Int) { guard let textView else { return } let range = NSRange(location: editedRange.location, length: editedRange.length - delta) @@ -339,26 +304,20 @@ extension Highlighter { visibleSet.insert(range: editedRange) } - guard let invalidatedIndexSet = await highlightProvider?.applyEdit( - textView: textView, - range: range, - delta: delta - ) else { - return - } - - let indexSet = invalidatedIndexSet - .union(IndexSet(integersIn: editedRange)) - // Only invalidate indices that are visible. - .intersection(visibleSet) + highlightProvider?.applyEdit(textView: textView, range: range, delta: delta) { [weak self] invalidIndexSet in + let indexSet = invalidIndexSet + .union(IndexSet(integersIn: editedRange)) + // Only invalidate indices that are visible. + .intersection(self?.visibleSet ?? IndexSet()) - for range in indexSet.rangeView { - invalidate(range: NSRange(range)) + for range in indexSet.rangeView { + self?.invalidate(range: NSRange(range)) + } } } - func storageWillEdit(editedRange: NSRange) async { + func storageWillEdit(editedRange: NSRange) { guard let textView else { return } - await highlightProvider?.willApplyEdit(textView: textView, range: editedRange) + highlightProvider?.willApplyEdit(textView: textView, range: editedRange) } } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift index 8421f1c6e..8d8444523 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/LanguageLayer.swift @@ -77,7 +77,7 @@ public class LanguageLayer: Hashable { parser.timeout = timeout ?? 0 let newTree = calculateNewState( - tree: self.tree, + tree: self.tree?.mutableCopy(), parser: self.parser, edit: edit, readBlock: readBlock @@ -150,12 +150,5 @@ public class LanguageLayer: Hashable { enum Error: Swift.Error, LocalizedError { case parserTimeout - - var localizedDescription: String { - switch self { - case .parserTimeout: - return "Parser Timed Out." - } - } } } diff --git a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift index 15f26d710..42ce2263b 100644 --- a/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditSourceEditor/TreeSitter/TreeSitterClient.swift @@ -13,14 +13,35 @@ import OSLog /// # TreeSitterClient /// -/// ``TreeSitterClient`` is an actor type that manages a tree-sitter syntax tree and provides an API for notifying that -/// tree of edits and querying the tree. +/// ``TreeSitterClient`` is an class that manages a tree-sitter syntax tree and provides an API for notifying that +/// tree of edits and querying the tree. This type also conforms to ``HighlightProviding`` to provide syntax +/// highlighting. /// -/// This type also conforms to ``HighlightProviding`` to provide syntax highlighting. +/// The APIs this object provides can perform either asynchronously or synchronously. All calls to this object must +/// first be dispatched from the main queue to ensure serial access to internal properties. Any synchronous methods +/// can throw an ``TreeSitterClient/Error/syncUnavailable`` error if an asynchronous or synchronous call is already +/// being made on the object. In those cases it is up to the caller to decide whether or not to retry asynchronously. /// -public actor TreeSitterClient: HighlightProviding { +/// The only exception to the above rule is the ``HighlightProviding`` conformance methods. The methods for that +/// implementation may return synchronously or asynchronously depending on a variety of factors such as document +/// length, edit length, highlight length and if the object is available for a synchronous call. +public final class TreeSitterClient: HighlightProviding { static let logger: Logger = Logger(subsystem: "com.CodeEdit.CodeEditSourceEditor", category: "TreeSitterClient") + /// The number of operations running or enqueued to run on the dispatch queue. This variable **must** only be + /// changed from the main thread or race conditions are very likely. + private var runningOperationCount = 0 + + /// The number of times the object has been set up. Used to cancel async tasks if + /// ``TreeSitterClient/setUp(textView:codeLanguage:)`` is called. + private var setUpCount = 0 + + /// The concurrent queue to perform operations on. + private let operationQueue = DispatchQueue( + label: "CodeEditSourceEditor.TreeSitter.EditQueue", + qos: .userInteractive + ) + // MARK: - Properties /// A callback to use to efficiently fetch portions of text. @@ -49,6 +70,19 @@ public actor TreeSitterClient: HighlightProviding { /// The timeout for parsers to re-check if a task is canceled. This constant represents the period between /// checks. static let parserTimeout: TimeInterval = 0.1 + + /// The maximum length of an edit before it must be processed asynchronously + static let maxSyncEditLength: Int = 1024 + + /// The maximum length a document can be before all queries and edits must be processed asynchronously. + static let maxSyncContentLength: Int = 1_000_000 + + /// The maximum length a query can be before it must be performed asynchronously. + static let maxSyncQueryLength: Int = 4096 + } + + public enum Error: Swift.Error { + case syncUnavailable } // MARK: - HighlightProviding @@ -58,18 +92,17 @@ public actor TreeSitterClient: HighlightProviding { /// - textView: The text view to use as a data source. /// A weak reference will be kept for the lifetime of this object. /// - codeLanguage: The language to use for parsing. - public func setUp(textView: TextView, codeLanguage: CodeLanguage) async { - self.readBlock = await textView.createReadBlock() - self.readCallback = await textView.createReadCallback() - - let task = Task.detached { - await self.setState( - language: codeLanguage, - readCallback: self.readCallback!, - readBlock: self.readBlock! - ) - } - await task.value + public func setUp(textView: TextView, codeLanguage: CodeLanguage) { + Self.logger.debug("TreeSitterClient setting up with language: \(codeLanguage.id.rawValue, privacy: .public)") + + self.readBlock = textView.createReadBlock() + self.readCallback = textView.createReadCallback() + + self.setState( + language: codeLanguage, + readCallback: self.readCallback!, + readBlock: self.readBlock! + ) } /// Sets the client's new state. @@ -81,10 +114,74 @@ public actor TreeSitterClient: HighlightProviding { language: CodeLanguage, readCallback: @escaping SwiftTreeSitter.Predicate.TextProvider, readBlock: @escaping Parser.ReadBlock - ) async { - self.state = TreeSitterState(codeLanguage: language, readCallback: readCallback, readBlock: readBlock) + ) { + setUpCount += 1 + performAsync { [weak self] in + self?.state = TreeSitterState(codeLanguage: language, readCallback: readCallback, readBlock: readBlock) + } + } + + // MARK: - Async Operations + + /// Performs the given operation asynchronously. + /// + /// All completion handlers passed to this function will be enqueued on the `operationQueue` dispatch queue, + /// ensuring serial access to this class. + /// + /// This function will handle ensuring balanced increment/decrements are made to the `runningOperationCount` in + /// a safe manner. + /// + /// - Note: While in debug mode, this method will throw an assertion failure if not called from the Main thread. + /// - Parameter operation: The operation to perform + private func performAsync(_ operation: @escaping () -> Void) { + assertMain() + runningOperationCount += 1 + let setUpCountCopy = setUpCount + operationQueue.async { [weak self] in + guard self != nil && self?.setUpCount == setUpCountCopy else { return } + operation() + DispatchQueue.main.async { + self?.runningOperationCount -= 1 + } + } } + /// Attempts to perform a synchronous operation on the client. + /// + /// The operation will be dispatched synchronously to the `operationQueue`, this function will return once the + /// operation is finished. + /// + /// - Note: While in debug mode, this method will throw an assertion failure if not called from the Main thread. + /// - Parameter operation: The operation to perform synchronously. + /// - Throws: Can throw an ``TreeSitterClient/Error/syncUnavailable`` error if it's determined that an async + /// operation is unsafe. + private func performSync(_ operation: @escaping () -> Void) throws { + assertMain() + + guard runningOperationCount == 0 else { + throw Error.syncUnavailable + } + + runningOperationCount += 1 + + operationQueue.sync { + operation() + } + + self.runningOperationCount -= 1 + } + + /// Assert that the caller is calling from the main thread. + private func assertMain() { +#if DEBUG + if !Thread.isMainThread { + assertionFailure("TreeSitterClient used from non-main queue. This will cause race conditions.") + } +#endif + } + + // MARK: - HighlightProviding + /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. /// - Parameters: @@ -92,26 +189,45 @@ public actor TreeSitterClient: HighlightProviding { /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - public func applyEdit(textView: TextView, range: NSRange, delta: Int) async -> IndexSet { + public func applyEdit(textView: TextView, range: NSRange, delta: Int, completion: @escaping (IndexSet) -> Void) { let oldEndPoint: Point + if self.oldEndPoint != nil { oldEndPoint = self.oldEndPoint! } else { - oldEndPoint = await textView.pointForLocation(range.max) ?? .zero + oldEndPoint = textView.pointForLocation(range.max) ?? .zero } + guard let edit = InputEdit( range: range, delta: delta, oldEndPoint: oldEndPoint, textView: textView ) else { - return IndexSet() + completion(IndexSet()) + return + } + + let operation = { [weak self] in + let invalidatedRanges = self?.applyEdit(edit: edit) ?? IndexSet() + completion(invalidatedRanges) + } + + do { + let longEdit = range.length > Constants.maxSyncEditLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + + if longEdit || longDocument { + + } + try performSync(operation) + } catch { + performAsync(operation) } - return applyEdit(edit: edit) } - public func willApplyEdit(textView: TextView, range: NSRange) async { - oldEndPoint = await textView.pointForLocation(range.max) + public func willApplyEdit(textView: TextView, range: NSRange) { + oldEndPoint = textView.pointForLocation(range.max) } /// Initiates a highlight query. @@ -119,7 +235,28 @@ public actor TreeSitterClient: HighlightProviding { /// - textView: The text view to use. /// - range: The range to limit the highlights to. /// - completion: Called when the query completes. - public func queryHighlightsFor(textView: TextView, range: NSRange) async -> [HighlightRange] { - return queryHighlightsForRange(range: range) + public func queryHighlightsFor( + textView: TextView, + range: NSRange, + completion: @escaping ([HighlightRange]) -> Void + ) { + let operation = { [weak self] in + let highlights = self?.queryHighlightsForRange(range: range) + DispatchQueue.main.async { + completion(highlights ?? []) + } + } + + do { + let longQuery = range.length > Constants.maxSyncQueryLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + + if longQuery || longDocument { + throw Error.syncUnavailable + } + try performSync(operation) + } catch { + performAsync(operation) + } } } diff --git a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift index c7bf6e0ab..425dbc81d 100644 --- a/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditSourceEditorTests/TreeSitterClientTests.swift @@ -25,11 +25,20 @@ final class TreeSitterClientTests: XCTestCase { client = TreeSitterClient() } - func test_clientSetup() async { - await client.setUp(textView: textView, codeLanguage: .swift) - - let primaryLanguage = await client.state?.primaryLayer.id - let layerCount = await client.state?.layers.count + func test_clientSetup() { + client.setUp(textView: textView, codeLanguage: .swift) + + let now = Date() + while client.state == nil && abs(now.timeIntervalSinceNow) < 5 { + usleep(1000) + } + + if abs(now.timeIntervalSinceNow) >= 5 { + XCTFail("Client took more than 5 seconds to set up.") + } + + let primaryLanguage = client.state?.primaryLayer.id + let layerCount = client.state?.layers.count XCTAssertEqual(primaryLanguage, .swift, "Client set up incorrect language") XCTAssertEqual(layerCount, 1, "Client set up too many layers") }