Merge pull request #16738 from atom/mb-tree-sitter-crlf-handling

Fix difference in CRLF line ending handling between tree-sitter and text-buffer

package.json

@@ -72,7 +72,7 @@
     "sinon": "1.17.4",
     "temp": "^0.8.3",
     "text-buffer": "13.11.8",
-    "tree-sitter": "^0.9.0",
+    "tree-sitter": "^0.9.1",
     "typescript-simple": "1.0.0",
     "underscore-plus": "^1.6.6",
     "winreg": "^1.2.1",

spec/tree-sitter-language-mode-spec.js

@@ -170,6 +170,49 @@ describe('TreeSitterLanguageMode', () => {
         [{text: ')', scopes: []}]
       ])
     })
+
+    it('handles edits after tokens that end between CR and LF characters (regression)', () => {
+      const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+        parser: 'tree-sitter-javascript',
+        scopes: {
+          'comment': 'comment',
+          'string': 'string',
+          'property_identifier': 'property',
+        }
+      })
+
+      buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
+
+      buffer.setText([
+        '// abc',
+        '',
+        'a("b").c'
+      ].join('\r\n'))
+
+      expectTokensToEqual(editor, [
+        [{text: '// abc', scopes: ['comment']}],
+        [{text: '', scopes: []}],
+        [
+          {text: 'a(', scopes: []},
+          {text: '"b"', scopes: ['string']},
+          {text: ').', scopes: []},
+          {text: 'c', scopes: ['property']}
+        ]
+      ])
+
+      buffer.insert([2, 0], '  ')
+      expectTokensToEqual(editor, [
+        [{text: '// abc', scopes: ['comment']}],
+        [{text: '', scopes: []}],
+        [
+          {text: '  ', scopes: ['whitespace']},
+          {text: 'a(', scopes: []},
+          {text: '"b"', scopes: ['string']},
+          {text: ').', scopes: []},
+          {text: 'c', scopes: ['property']}
+        ]
+      ])
+    })
   })
 
   describe('folding', () => {

src/tree-sitter-language-mode.js

@@ -495,16 +495,22 @@ class TreeSitterHighlightIterator {
 class TreeSitterTextBufferInput {
   constructor (buffer) {
     this.buffer = buffer
-    this.seek(0)
+    this.position = {row: 0, column: 0}
+    this.isBetweenCRLF = false
   }
 
-  seek (characterIndex) {
-    this.position = this.buffer.positionForCharacterIndex(characterIndex)
+  seek (offset, position) {
+    this.position = position
+    this.isBetweenCRLF = this.position.column > this.buffer.lineLengthForRow(this.position.row)
   }
 
   read () {
-    const endPosition = this.buffer.clipPosition(this.position.traverse({row: 1000, column: 0}))
-    const text = this.buffer.getTextInRange([this.position, endPosition])
+    const endPosition = this.buffer.clipPosition(new Point(this.position.row + 1000, 0))
+    let text = this.buffer.getTextInRange([this.position, endPosition])
+    if (this.isBetweenCRLF) {
+      text = text.slice(1)
+      this.isBetweenCRLF = false
+    }
     this.position = endPosition
     return text
   }