Add initial TreeSitterLanguageMode implementation

Much of this is from the tree-sitter-syntax package.
Also, add a dependency on the tree-sitter module.

package.json

@@ -71,6 +71,7 @@
     "sinon": "1.17.4",
     "temp": "^0.8.3",
     "text-buffer": "13.9.2",
+    "tree-sitter": "0.7.4",
     "typescript-simple": "1.0.0",
     "underscore-plus": "^1.6.6",
     "winreg": "^1.2.1",

spec/syntax-scope-map-spec.js

@@ -0,0 +1,77 @@
+const SyntaxScopeMap = require('../src/syntax-scope-map')
+
+describe('SyntaxScopeMap', () => {
+  it('can match immediate child selectors', () => {
+    const map = new SyntaxScopeMap({
+      'a > b > c': 'x',
+      'b > c': 'y',
+      'c': 'z'
+    })
+
+    expect(map.get(['a', 'b', 'c'], [0, 0, 0])).toBe('x')
+    expect(map.get(['d', 'b', 'c'], [0, 0, 0])).toBe('y')
+    expect(map.get(['d', 'e', 'c'], [0, 0, 0])).toBe('z')
+    expect(map.get(['e', 'c'], [0, 0, 0])).toBe('z')
+    expect(map.get(['c'], [0, 0, 0])).toBe('z')
+    expect(map.get(['d'], [0, 0, 0])).toBe(undefined)
+  })
+
+  it('can match :nth-child pseudo-selectors on leaves', () => {
+    const map = new SyntaxScopeMap({
+      'a > b': 'w',
+      'a > b:nth-child(1)': 'x',
+      'b': 'y',
+      'b:nth-child(2)': 'z'
+    })
+
+    expect(map.get(['a', 'b'], [0, 0])).toBe('w')
+    expect(map.get(['a', 'b'], [0, 1])).toBe('x')
+    expect(map.get(['a', 'b'], [0, 2])).toBe('w')
+    expect(map.get(['b'], [0])).toBe('y')
+    expect(map.get(['b'], [1])).toBe('y')
+    expect(map.get(['b'], [2])).toBe('z')
+  })
+
+  it('can match :nth-child pseudo-selectors on interior nodes', () => {
+    const map = new SyntaxScopeMap({
+      'b:nth-child(1) > c': 'w',
+      'a > b > c': 'x',
+      'a > b:nth-child(2) > c': 'y'
+    })
+
+    expect(map.get(['b', 'c'], [0, 0])).toBe(undefined)
+    expect(map.get(['b', 'c'], [1, 0])).toBe('w')
+    expect(map.get(['a', 'b', 'c'], [1, 0, 0])).toBe('x')
+    expect(map.get(['a', 'b', 'c'], [1, 2, 0])).toBe('y')
+  })
+
+  it('allows anonymous tokens to be referred to by their string value', () => {
+    const map = new SyntaxScopeMap({
+      '"b"': 'w',
+      'a > "b"': 'x',
+      'a > "b":nth-child(1)': 'y'
+    })
+
+    expect(map.get(['b'], [0], true)).toBe(undefined)
+    expect(map.get(['b'], [0], false)).toBe('w')
+    expect(map.get(['a', 'b'], [0, 0], false)).toBe('x')
+    expect(map.get(['a', 'b'], [0, 1], false)).toBe('y')
+  })
+
+  it('supports the wildcard selector', () => {
+    const map = new SyntaxScopeMap({
+      '*': 'w',
+      'a > *': 'x',
+      'a > *:nth-child(1)': 'y',
+      'a > *:nth-child(1) > b': 'z'
+    })
+
+    expect(map.get(['b'], [0])).toBe('w')
+    expect(map.get(['c'], [0])).toBe('w')
+    expect(map.get(['a', 'b'], [0, 0])).toBe('x')
+    expect(map.get(['a', 'b'], [0, 1])).toBe('y')
+    expect(map.get(['a', 'c'], [0, 1])).toBe('y')
+    expect(map.get(['a', 'c', 'b'], [0, 1, 1])).toBe('z')
+    expect(map.get(['a', 'c', 'b'], [0, 2, 1])).toBe('w')
+  })
+})

src/syntax-scope-map.js

@@ -0,0 +1,178 @@
+const parser = require('postcss-selector-parser')
+
+module.exports =
+class SyntaxScopeMap {
+  constructor (scopeNamesBySelector) {
+    this.namedScopeTable = {}
+    this.anonymousScopeTable = {}
+    for (let selector in scopeNamesBySelector) {
+      this.addSelector(selector, scopeNamesBySelector[selector])
+    }
+    setTableDefaults(this.namedScopeTable)
+    setTableDefaults(this.anonymousScopeTable)
+  }
+
+  addSelector (selector, scopeName) {
+    parser((parseResult) => {
+      for (let selectorNode of parseResult.nodes) {
+        let currentTable = null
+        let currentIndexValue = null
+
+        for (let i = selectorNode.nodes.length - 1; i >= 0; i--) {
+          const termNode = selectorNode.nodes[i]
+
+          switch (termNode.type) {
+            case 'tag':
+              if (!currentTable) currentTable = this.namedScopeTable
+              if (!currentTable[termNode.value]) currentTable[termNode.value] = {}
+              currentTable = currentTable[termNode.value]
+              if (currentIndexValue != null) {
+                if (!currentTable.indices) currentTable.indices = {}
+                if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {}
+                currentTable = currentTable.indices[currentIndexValue]
+                currentIndexValue = null
+              }
+              break
+
+            case 'string':
+              if (!currentTable) currentTable = this.anonymousScopeTable
+              const value = termNode.value.slice(1, -1)
+              if (!currentTable[value]) currentTable[value] = {}
+              currentTable = currentTable[value]
+              if (currentIndexValue != null) {
+                if (!currentTable.indices) currentTable.indices = {}
+                if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {}
+                currentTable = currentTable.indices[currentIndexValue]
+                currentIndexValue = null
+              }
+              break
+
+            case 'universal':
+              if (currentTable) {
+                if (!currentTable['*']) currentTable['*'] = {}
+                currentTable = currentTable['*']
+              } else {
+                if (!this.namedScopeTable['*']) {
+                  this.namedScopeTable['*'] = this.anonymousScopeTable['*'] = {}
+                }
+                currentTable = this.namedScopeTable['*']
+              }
+              if (currentIndexValue != null) {
+                if (!currentTable.indices) currentTable.indices = {}
+                if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {}
+                currentTable = currentTable.indices[currentIndexValue]
+                currentIndexValue = null
+              }
+              break
+
+            case 'combinator':
+              if (currentIndexValue != null) {
+                rejectSelector(selector)
+              }
+
+              if (termNode.value === '>') {
+                if (!currentTable.parents) currentTable.parents = {}
+                currentTable = currentTable.parents
+              } else {
+                rejectSelector(selector)
+              }
+              break
+
+            case 'pseudo':
+              if (termNode.value === ':nth-child') {
+                currentIndexValue = termNode.nodes[0].nodes[0].value
+              } else {
+                rejectSelector(selector)
+              }
+              break
+
+            default:
+              rejectSelector(selector)
+          }
+        }
+
+        currentTable.scopeName = scopeName
+      }
+    }).process(selector)
+  }
+
+  get (nodeTypes, childIndices, leafIsNamed = true) {
+    let result
+    let i = nodeTypes.length - 1
+    let currentTable = leafIsNamed
+      ? this.namedScopeTable[nodeTypes[i]]
+      : this.anonymousScopeTable[nodeTypes[i]]
+
+    if (!currentTable) currentTable = this.namedScopeTable['*']
+
+    while (currentTable) {
+      if (currentTable.indices && currentTable.indices[childIndices[i]]) {
+        currentTable = currentTable.indices[childIndices[i]]
+      }
+
+      if (currentTable.scopeName) {
+        result = currentTable.scopeName
+      }
+
+      if (i === 0) break
+      i--
+      currentTable = currentTable.parents && (
+        currentTable.parents[nodeTypes[i]] ||
+        currentTable.parents['*']
+      )
+    }
+
+    return result
+  }
+}
+
+function setTableDefaults (table) {
+  const defaultTypeTable = table['*']
+
+  for (let type in table) {
+    let typeTable = table[type]
+    if (typeTable === defaultTypeTable) continue
+
+    if (defaultTypeTable) {
+      mergeTable(typeTable, defaultTypeTable)
+    }
+
+    if (typeTable.parents) {
+      setTableDefaults(typeTable.parents)
+    }
+
+    for (let key in typeTable.indices) {
+      const indexTable = typeTable.indices[key]
+      mergeTable(indexTable, typeTable, false)
+      if (indexTable.parents) {
+        setTableDefaults(indexTable.parents)
+      }
+    }
+  }
+}
+
+function mergeTable (table, defaultTable, mergeIndices = true) {
+  if (mergeIndices && defaultTable.indices) {
+    if (!table.indices) table.indices = {}
+    for (let key in defaultTable.indices) {
+      if (!table.indices[key]) table.indices[key] = {}
+      mergeTable(table.indices[key], defaultTable.indices[key])
+    }
+  }
+
+  if (defaultTable.parents) {
+    if (!table.parents) table.parents = {}
+    for (let key in defaultTable.parents) {
+      if (!table.parents[key]) table.parents[key] = {}
+      mergeTable(table.parents[key], defaultTable.parents[key])
+    }
+  }
+
+  if (defaultTable.scopeName && !table.scopeName) {
+    table.scopeName = defaultTable.scopeName
+  }
+}
+
+function rejectSelector (selector) {
+  throw new TypeError(`Unsupported selector '${selector}'`)
+}

src/tree-sitter-grammar.js

@@ -0,0 +1,74 @@
+const path = require('path')
+const SyntaxScopeMap = require('./syntax-scope-map')
+const Module = require('module')
+const {OnigRegExp} = require('oniguruma')
+
+module.exports =
+class TreeSitterGrammar {
+  constructor (registry, filePath, params) {
+    this.registry = registry
+    this.id = params.id
+    this.name = params.name
+
+    this.foldConfig = params.folds || {}
+    if (!this.foldConfig.delimiters) this.foldConfig.delimiters = []
+    if (!this.foldConfig.tokens) this.foldConfig.tokens = []
+
+    this.commentStrings = {
+      commentStartString: params.comments && params.comments.start,
+      commentEndString: params.comments && params.comments.end
+    }
+
+    const scopeSelectors = {}
+    for (const key of Object.keys(params.scopes)) {
+      scopeSelectors[key] = params.scopes[key]
+        .split('.')
+        .map(s => `syntax--${s}`)
+        .join(' ')
+    }
+
+    this.scopeMap = new SyntaxScopeMap(scopeSelectors)
+    this.fileTypes = params.fileTypes
+
+    // TODO - When we upgrade to a new enough version of node, use `require.resolve`
+    // with the new `paths` option instead of this private API.
+    const languageModulePath = Module._resolveFilename(params.parser, {
+      id: filePath,
+      filename: filePath,
+      paths: Module._nodeModulePaths(path.dirname(filePath))
+    })
+
+    this.languageModule = require(languageModulePath)
+    this.firstLineRegex = new OnigRegExp(params.firstLineMatch)
+    this.scopesById = new Map()
+    this.idsByScope = {}
+    this.nextScopeId = 256 + 1
+    this.registration = null
+  }
+
+  idForScope (scope) {
+    let id = this.idsByScope[scope]
+    if (!id) {
+      id = this.nextScopeId += 2
+      this.idsByScope[scope] = id
+      this.scopesById.set(id, scope)
+    }
+    return id
+  }
+
+  classNameForScopeId (id) {
+    return this.scopesById.get(id)
+  }
+
+  get scopeName () {
+    return this.id
+  }
+
+  activate () {
+    this.registration = this.registry.addGrammar(this)
+  }
+
+  deactivate () {
+    if (this.registration) this.registration.dispose()
+  }
+}