tree-sitter step 2: query API and basic highlighting

runtime/doc/lua.txt

@@ -594,6 +594,102 @@ tsnode:named_descendant_for_range(start_row, start_col, end_row, end_col)
 	Get the smallest named node within this node that spans the given
 	range of (row, column) positions
 
+Query methods						*lua-treesitter-query*
+
+Tree-sitter queries are supported, with some limitations. Currently, the only
+supported match predicate is `eq?` (both comparing a capture against a string
+and two captures against each other).
+
+vim.treesitter.parse_query(lang, query)
+						*vim.treesitter.parse_query(()*
+	Parse the query as a string. (If the query is in a file, the caller
+        should read the contents into a string before calling).
+
+query:iter_captures(node, bufnr, start_row, end_row)
+							*query:iter_captures()*
+	Iterate over all captures from all matches inside a `node`.
+	`bufnr` is needed if the query contains predicates, then the caller
+	must ensure to use a freshly parsed tree consistent with the current
+	text of the buffer. `start_row` and `end_row` can be used to limit
+	matches inside a row range (this is typically used with root node
+	as the node, i e to get syntax highlight matches in the current
+	viewport)
+
+	The iterator returns two values, a numeric id identifying the capture
+	and the captured node. The following example shows how to get captures
+	by name:
+>
+	for id, node in query:iter_captures(tree:root(), bufnr, first, last) do
+	  local name = query.captures[id] -- name of the capture in the query
+	  -- typically useful info about the node:
+	  local type = node:type() -- type of the captured node
+	  local row1, col1, row2, col2 = node:range() -- range of the capture
+	  ... use the info here ...
+	end
+<
+query:iter_matches(node, bufnr, start_row, end_row)
+							*query:iter_matches()*
+	Iterate over all matches within a node. The arguments are the same as
+	for |query:iter_captures()| but the iterated values are different:
+	an (1-based) index of the pattern in the query, and a table mapping
+	capture indices to nodes. If the query has more than one pattern
+	the capture table might be sparse, and e.g. `pairs` should be used and not
+	`ipairs`. Here an example iterating over all captures in
+	every match:
+>
+	for pattern, match in cquery:iter_matches(tree:root(), bufnr, first, last) do
+	  for id,node in pairs(match) do
+	    local name = query.captures[id]
+	    -- `node` was captured by the `name` capture in the match
+	    ... use the info here ...
+	  end
+	end
+>
+Treesitter syntax highlighting (WIP)		    *lua-treesitter-highlight*
+
+NOTE: This is a partially implemented feature, and not usable as a default
+solution yet. What is documented here is a temporary interface indented
+for those who want to experiment with this feature and contribute to
+its development.
+
+Highlights are defined in the same query format as in the tree-sitter highlight
+crate, which some limitations and additions. Set a highlight query for a
+buffer with this code: >
+
+    local query = [[
+      "for" @keyword
+      "if" @keyword
+      "return" @keyword
+
+      (string_literal) @string
+      (number_literal) @number
+      (comment) @comment
+
+      (preproc_function_def name: (identifier) @function)
+
+      ; ... more definitions
+    ]]
+
+    highlighter = vim.treesitter.TSHighlighter.new(query, bufnr, lang)
+    -- alternatively, to use the current buffer and its filetype:
+    -- highlighter = vim.treesitter.TSHighlighter.new(query)
+
+    -- Don't recreate the highlighter for the same buffer, instead
+    -- modify the query like this:
+    local query2 = [[ ... ]]
+    highlighter:set_query(query2)
+
+As mentioned above the supported predicate is currently only `eq?`. `match?`
+predicates behave like matching always fails. As an addition a capture which
+begin with an upper-case letter like `@WarningMsg` will map directly to this
+highlight group, if defined. Also if the predicate begins with upper-case and
+contains a dot only the part before the first will be interpreted as the
+highlight group. As an example, this warns of a binary expression with two
+identical identifiers, highlighting both as |hl-WarningMsg|: >
+
+    ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right)
+     (eq? @WarningMsg.left @WarningMsg.right))
+
 ------------------------------------------------------------------------------
 VIM							*lua-builtin*
 

runtime/lua/vim/treesitter.lua

@@ -12,9 +12,13 @@ function Parser:parse()
   if self.valid then
     return self.tree
   end
-  self.tree = self._parser:parse_buf(self.bufnr)
+  local changes
+  self.tree, changes = self._parser:parse_buf(self.bufnr)
   self.valid = true
-  return self.tree
+  for _, cb in ipairs(self.change_cbs) do
+    cb(changes)
+  end
+  return self.tree, changes
 end
 
 function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_size)
@@ -26,17 +30,28 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_
   self.valid = false
 end
 
-local module = {
+local M = {
   add_language=vim._ts_add_language,
   inspect_language=vim._ts_inspect_language,
+  parse_query = vim._ts_parse_query,
 }
 
-function module.create_parser(bufnr, ft, id)
+setmetatable(M, {
+  __index = function (t, k)
+      if k == "TSHighlighter" then
+        t[k] = require'vim.tshighlighter'
+        return t[k]
+      end
+   end
+ })
+
+function M.create_parser(bufnr, ft, id)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  local self = setmetatable({bufnr=bufnr, valid=false}, Parser)
+  local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser)
   self._parser = vim._create_ts_parser(ft)
+  self.change_cbs = {}
   self:parse()
     -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
     -- using it.
@@ -55,7 +70,7 @@ function module.create_parser(bufnr, ft, id)
   return self
 end
 
-function module.get_parser(bufnr, ft)
+function M.get_parser(bufnr, ft, cb)
   if bufnr == nil or bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
@@ -65,9 +80,98 @@ function module.get_parser(bufnr, ft)
   local id = tostring(bufnr)..'_'..ft
 
   if parsers[id] == nil then
-    parsers[id] = module.create_parser(bufnr, ft, id)
+    parsers[id] = M.create_parser(bufnr, ft, id)
+  end
+  if cb ~= nil then
+    table.insert(parsers[id].change_cbs, cb)
   end
   return parsers[id]
 end
 
-return module
+-- query: pattern matching on trees
+-- predicate matching is implemented in lua
+local Query = {}
+Query.__index = Query
+
+function M.parse_query(lang, query)
+  local self = setmetatable({}, Query)
+  self.query = vim._ts_parse_query(lang, query)
+  self.info = self.query:inspect()
+  self.captures = self.info.captures
+  return self
+end
+
+local function get_node_text(node, bufnr)
+  local start_row, start_col, end_row, end_col = node:range()
+  if start_row ~= end_row then
+    return nil
+  end
+  local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1]
+  return string.sub(line, start_col+1, end_col)
+end
+
+local function match_preds(match, preds, bufnr)
+  for _, pred in pairs(preds) do
+    if pred[1] == "eq?" then
+      local node = match[pred[2]]
+      local node_text = get_node_text(node, bufnr)
+
+      local str
+      if type(pred[3]) == "string" then
+        -- (eq? @aa "foo")
+        str = pred[3]
+      else
+        -- (eq? @aa @bb)
+        str = get_node_text(match[pred[3]], bufnr)
+      end
+
+      if node_text ~= str or str == nil then
+        return false
+      end
+    else
+      return false
+    end
+  end
+  return true
+end
+
+function Query:iter_captures(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,true,start,stop)
+  local function iter()
+    local capture, captured_node, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[match.pattern]
+      local active = match_preds(match, preds, bufnr)
+      match.active = active
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return capture, captured_node
+  end
+  return iter
+end
+
+function Query:iter_matches(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,false,start,stop)
+  local function iter()
+    local pattern, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[pattern]
+      local active = (not preds) or match_preds(match, preds, bufnr)
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return pattern, match
+  end
+  return iter
+end
+
+return M