Revert "Merge pull request #6757 from atom/ns-less-memory-for-tokens"

This reverts commit 0cd1f11, reversing
changes made to d75d202.

Conflicts:
	package.json

package.json

@@ -32,7 +32,7 @@
     "delegato": "^1",
     "emissary": "^1.3.3",
     "event-kit": "^1.2.0",
-    "first-mate": "^4.1.4",
+    "first-mate": "^3.1",
     "fs-plus": "^2.8.0",
     "fstream": "0.1.24",
     "fuzzaldrin": "^2.1",
@@ -151,7 +151,7 @@
     "language-ruby": "0.54.0",
     "language-ruby-on-rails": "0.21.0",
     "language-sass": "0.38.0",
-    "language-shellscript": "0.15.0",
+    "language-shellscript": "0.14.0",
     "language-source": "0.9.0",
     "language-sql": "0.15.0",
     "language-text": "0.6.0",

spec/text-editor-presenter-spec.coffee

@@ -670,59 +670,39 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 4), {
             screenRow: 4
             text: line4.text
-            tags: line4.tags
-            specialTokens: line4.specialTokens
-            firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex
-            invisibles: line4.invisibles
+            tokens: line4.tokens
             top: 10 * 4
           }
 
           line5 = editor.tokenizedLineForScreenRow(5)
           expectValues lineStateForScreenRow(presenter, 5), {
             screenRow: 5
             text: line5.text
-            tags: line5.tags
-            specialTokens: line5.specialTokens
-            firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex
-            invisibles: line5.invisibles
+            tokens: line5.tokens
             top: 10 * 5
           }
 
           line6 = editor.tokenizedLineForScreenRow(6)
           expectValues lineStateForScreenRow(presenter, 6), {
             screenRow: 6
             text: line6.text
-            tags: line6.tags
-            specialTokens: line6.specialTokens
-            firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex
-            invisibles: line6.invisibles
+            tokens: line6.tokens
             top: 10 * 6
           }
 
           line7 = editor.tokenizedLineForScreenRow(7)
           expectValues lineStateForScreenRow(presenter, 7), {
             screenRow: 7
             text: line7.text
-            tags: line7.tags
-            specialTokens: line7.specialTokens
-            firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex
-            invisibles: line7.invisibles
+            tokens: line7.tokens
             top: 10 * 7
           }
 
           line8 = editor.tokenizedLineForScreenRow(8)
           expectValues lineStateForScreenRow(presenter, 8), {
             screenRow: 8
             text: line8.text
-            tags: line8.tags
-            specialTokens: line8.specialTokens
-            firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex
-            invisibles: line8.invisibles
+            tokens: line8.tokens
             top: 10 * 8
           }
 
@@ -817,19 +797,19 @@ describe "TextEditorPresenter", ->
           line1 = editor.tokenizedLineForScreenRow(1)
           expectValues lineStateForScreenRow(presenter, 1), {
             text: line1.text
-            tags: line1.tags
+            tokens: line1.tokens
           }
 
           line2 = editor.tokenizedLineForScreenRow(2)
           expectValues lineStateForScreenRow(presenter, 2), {
             text: line2.text
-            tags: line2.tags
+            tokens: line2.tokens
           }
 
           line3 = editor.tokenizedLineForScreenRow(3)
           expectValues lineStateForScreenRow(presenter, 3), {
             text: line3.text
-            tags: line3.tags
+            tokens: line3.tokens
           }
 
         it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", ->

spec/text-editor-spec.coffee

@@ -4110,9 +4110,8 @@ describe "TextEditor", ->
 
       runs ->
         grammar = atom.grammars.selectGrammar("text.js")
-        {line, tags} = grammar.tokenizeLine("var i; // http://github.com")
+        {tokens} = grammar.tokenizeLine("var i; // http://github.com")
 
-        tokens = atom.grammars.decodeTokens(line, tags)
         expect(tokens[0].value).toBe "var"
         expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"]
 

spec/tokenized-buffer-spec.coffee

@@ -296,6 +296,14 @@ describe "TokenizedBuffer", ->
           expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
           expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy()
 
+      describe ".findOpeningBracket(closingBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
+
+      describe ".findClosingBracket(startBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
+
       it "tokenizes leading whitespace based on the new tab length", ->
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe "  "
@@ -572,7 +580,7 @@ describe "TokenizedBuffer", ->
 
     describe "when the selector matches a run of multiple tokens at the position", ->
       it "returns the range covered by all contigous tokens (within a single line)", ->
-        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]]
+        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]]
 
   describe "when the editor.tabLength config value changes", ->
     it "updates the tab length of the tokenized lines", ->
@@ -689,6 +697,22 @@ describe "TokenizedBuffer", ->
       expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
       expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
 
+    it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", ->
+      atom.config.set("editor.showInvisibles", true)
+      atom.config.set("editor.invisibles", space: 'S')
+      buffer.setText(" token ")
+      fullyTokenize(tokenizedBuffer)
+      token = tokenizedBuffer.tokenizedLines[0].tokens[0]
+
+      [leftToken, rightToken] = token.splitAt(1)
+      expect(leftToken.hasInvisibleCharacters).toBe true
+      expect(leftToken.firstNonWhitespaceIndex).toBe 1
+      expect(leftToken.firstTrailingWhitespaceIndex).toBe null
+
+      expect(leftToken.hasInvisibleCharacters).toBe true
+      expect(rightToken.firstNonWhitespaceIndex).toBe null
+      expect(rightToken.firstTrailingWhitespaceIndex).toBe 5
+
   describe ".indentLevel on tokenized lines", ->
     beforeEach ->
       buffer = atom.project.bufferForPathSync('sample.js')
@@ -728,7 +752,7 @@ describe "TokenizedBuffer", ->
       it "updates empty line indent guides when the empty line is the last line", ->
         buffer.insert([12, 2], '\n')
 
-        # The newline and the tab need to be in two different operations to surface the bug
+        # The newline and he tab need to be in two different operations to surface the bug
         buffer.insert([12, 0], '  ')
         expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1
 

spec/tokenized-line-spec.coffee

@@ -17,3 +17,24 @@ describe "TokenizedLine", ->
     it "returns false when the line is not only whitespace", ->
       expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false
       expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false
+
+  describe "::getScopeTree()", ->
+    it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", ->
+      [tokens, tokenIndex] = []
+
+      ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) ->
+        if scopeTree.children?
+          for child in scopeTree.children
+            ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope]))
+        else
+          expect(scopeTree).toBe tokens[tokenIndex++]
+          expect(scopeDescriptor).toEqual scopeTree.scopes
+
+      waitsForPromise ->
+        atom.project.open('coffee.coffee').then (o) -> editor = o
+
+      runs ->
+        tokenIndex = 0
+        tokens = editor.tokenizedLineForScreenRow(1).tokens
+        scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree()
+        ensureValidScopeTree(scopeTree)

src/display-buffer.coffee

@@ -2,14 +2,14 @@ _ = require 'underscore-plus'
 Serializable = require 'serializable'
 {CompositeDisposable, Emitter} = require 'event-kit'
 {Point, Range} = require 'text-buffer'
-Grim = require 'grim'
 TokenizedBuffer = require './tokenized-buffer'
 RowMap = require './row-map'
 Fold = require './fold'
 Model = require './model'
 Token = require './token'
 Decoration = require './decoration'
 Marker = require './marker'
+Grim = require 'grim'
 
 class BufferToScreenConversionError extends Error
   constructor: (@message, @metadata) ->
@@ -659,19 +659,16 @@ class DisplayBuffer extends Model
     top = targetRow * @lineHeightInPixels
     left = 0
     column = 0
-
-    iterator = @tokenizedLineForScreenRow(targetRow).getTokenIterator()
-    while iterator.next()
-      charWidths = @getScopedCharWidths(iterator.getScopes())
+    for token in @tokenizedLineForScreenRow(targetRow).tokens
+      charWidths = @getScopedCharWidths(token.scopes)
       valueIndex = 0
-      value = iterator.getText()
-      while valueIndex < value.length
-        if iterator.isPairedCharacter()
-          char = value
+      while valueIndex < token.value.length
+        if token.hasPairedCharacter
+          char = token.value.substr(valueIndex, 2)
           charLength = 2
           valueIndex += 2
         else
-          char = value[valueIndex]
+          char = token.value[valueIndex]
           charLength = 1
           valueIndex++
 
@@ -692,19 +689,16 @@ class DisplayBuffer extends Model
 
     left = 0
     column = 0
-
-    iterator = @tokenizedLineForScreenRow(row).getTokenIterator()
-    while iterator.next()
-      charWidths = @getScopedCharWidths(iterator.getScopes())
-      value = iterator.getText()
+    for token in @tokenizedLineForScreenRow(row).tokens
+      charWidths = @getScopedCharWidths(token.scopes)
       valueIndex = 0
-      while valueIndex < value.length
-        if iterator.isPairedCharacter()
-          char = value
+      while valueIndex < token.value.length
+        if token.hasPairedCharacter
+          char = token.value.substr(valueIndex, 2)
           charLength = 2
           valueIndex += 2
         else
-          char = value[valueIndex]
+          char = token.value[valueIndex]
           charLength = 1
           valueIndex++
 

src/language-mode.coffee

@@ -242,9 +242,8 @@ class LanguageMode
     @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options)
 
   suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) ->
-    iterator = tokenizedLine.getTokenIterator()
-    iterator.next()
-    scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes())
+    scopes = tokenizedLine.tokens[0].scopes
+    scopeDescriptor = new ScopeDescriptor({scopes})
 
     currentIndentLevel = @editor.indentationForBufferRow(bufferRow)
     return currentIndentLevel unless increaseIndentRegex = @increaseIndentRegexForScopeDescriptor(scopeDescriptor)