feat(treesitter): support URLs (#27132)

Tree-sitter queries can add URLs to a capture using the `#set!`
directive, e.g.

  (inline_link
    (link_text) @text.reference
    (link_destination) @text.uri
    (#set! @text.reference "url" @text.uri))

The pattern above is included by default in the `markdown_inline`
highlight query so that users with supporting terminals will see
hyperlinks. For now, this creates a hyperlink for *all* Markdown URLs of
the pattern [link text](link url), even if `link url` does not contain
a valid protocol (e.g. if `link url` is a path to a file). We may wish to
change this in the future to only linkify when the URL has a valid
protocol scheme, but for now we delegate handling this to the terminal
emulator.

In order to support directives which reference other nodes, the
highlighter must be updated to use `iter_matches` rather than
`iter_captures`. The former provides the `match` table which maps
capture IDs to nodes. However, this has its own challenges:

- `iter_matches` does not guarantee the order in which patterns are
  iterated matches the order in the query file. So we must enforce
  ordering manually using "subpriorities" (#27131). The pattern index of
  each match dictates the extmark's subpriority.
- When injections are used, the highlighter contains multiple trees. The
  pattern indices of each tree must be offset relative to the maximum
  pattern index from all previous trees to ensure that extmarks appear
  in the correct order.
- The `iter_captures` implementation currently has a bug where the
  "match" table is only returned for the first capture within a pattern
  (see #27274). This bug means that `#set!` directives in a query
  apply only to the first capture within a pattern. Unfortunately, many
  queries in the wild have come to depend on this behavior.
  `iter_matches` does not share this flaw, so switching to
  `iter_matches` exposed bugs in existing highlight queries. These
  queries have been updated in this repo, but may still need to be
  updated by users. The `#set!` directive applies to the _entire_ query
  pattern when used without a capture argument. To make `#set!`
  apply only to a single capture, the capture must be given as an
  argument.

runtime/doc/news.txt

@@ -254,6 +254,9 @@ The following new APIs and features were added.
     indexing.
   • |:InspectTree| shows root nodes
   • |:InspectTree| now supports |folding|
+  • The `#set!` directive can set the "url" property of a node to have the
+    node emit a hyperlink. Hyperlinks are UI specific: in the TUI, the OSC 8
+    control sequence is used.
 
 • |vim.ui.open()| opens URIs using the system default handler (macOS `open`,
   Windows `explorer`, Linux `xdg-open`, etc.)

runtime/lua/vim/treesitter/highlighter.lua

@@ -4,7 +4,7 @@ local Range = require('vim.treesitter._range')
 
 local ns = api.nvim_create_namespace('treesitter/highlighter')
 
----@alias vim.treesitter.highlighter.Iter fun(end_line: integer|nil): integer, TSNode, vim.treesitter.query.TSMetadata
+---@alias vim.treesitter.highlighter.Iter fun(): integer, table<integer, TSNode[]>, vim.treesitter.query.TSMetadata
 
 ---@class (private) vim.treesitter.highlighter.Query
 ---@field private _query vim.treesitter.Query?
@@ -248,6 +248,13 @@ end
 ---@param line integer
 ---@param is_spell_nav boolean
 local function on_line_impl(self, buf, line, is_spell_nav)
+  -- Track the maximum pattern index encountered in each tree. For subsequent
+  -- trees, the subpriority passed to nvim_buf_set_extmark is offset by the
+  -- largest pattern index from the prior tree. This ensures that extmarks
+  -- from subsequent trees always appear "on top of" extmarks from previous
+  -- trees (e.g. injections should always appear over base highlights).
+  local pattern_offset = 0
+
   self:for_each_highlight_state(function(state)
     local root_node = state.tstree:root()
     local root_start_row, _, root_end_row, _ = root_node:range()
@@ -258,22 +265,24 @@ local function on_line_impl(self, buf, line, is_spell_nav)
     end
 
     if state.iter == nil or state.next_row < line then
-      state.iter =
-        state.highlighter_query:query():iter_captures(root_node, self.bufnr, line, root_end_row + 1)
+      state.iter = state.highlighter_query
+        :query()
+        :iter_matches(root_node, self.bufnr, line, root_end_row + 1, { all = true })
     end
 
+    local max_pattern_index = -1
     while line >= state.next_row do
-      local capture, node, metadata = state.iter(line)
+      local pattern, match, metadata = state.iter()
 
-      local range = { root_end_row + 1, 0, root_end_row + 1, 0 }
-      if node then
-        range = vim.treesitter.get_range(node, buf, metadata and metadata[capture])
+      if pattern and pattern > max_pattern_index then
+        max_pattern_index = pattern
       end
-      local start_row, start_col, end_row, end_col = Range.unpack4(range)
 
-      if capture then
-        local hl = state.highlighter_query:get_hl_from_capture(capture)
+      if not match then
+        state.next_row = root_end_row + 1
+      end
 
+      for capture, nodes in pairs(match or {}) do
         local capture_name = state.highlighter_query:query().captures[capture]
         local spell = nil ---@type boolean?
         if capture_name == 'spell' then
@@ -282,28 +291,60 @@ local function on_line_impl(self, buf, line, is_spell_nav)
           spell = false
         end
 
+        local hl = state.highlighter_query:get_hl_from_capture(capture)
+
         -- Give nospell a higher priority so it always overrides spell captures.
         local spell_pri_offset = capture_name == 'nospell' and 1 or 0
 
-        if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
-          local priority = (tonumber(metadata.priority) or vim.highlight.priorities.treesitter)
-            + spell_pri_offset
-          api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
-            end_line = end_row,
-            end_col = end_col,
-            hl_group = hl,
-            ephemeral = true,
-            priority = priority,
-            conceal = metadata.conceal,
-            spell = spell,
-          })
+        -- The "priority" attribute can be set at the pattern level or on a particular capture
+        local priority = (
+          tonumber(metadata.priority or metadata[capture] and metadata[capture].priority)
+          or vim.highlight.priorities.treesitter
+        ) + spell_pri_offset
+
+        local url = metadata[capture] and metadata[capture].url ---@type string|number|nil
+        if type(url) == 'number' then
+          if match and match[url] then
+            -- Assume there is only one matching node. If there is more than one, take the URL
+            -- from the first.
+            local other_node = match[url][1]
+            url = vim.treesitter.get_node_text(other_node, buf, {
+              metadata = metadata[url],
+            })
+          else
+            url = nil
+          end
         end
-      end
 
-      if start_row > line then
-        state.next_row = start_row
+        -- The "conceal" attribute can be set at the pattern level or on a particular capture
+        local conceal = metadata.conceal or metadata[capture] and metadata[capture].conceal
+
+        for _, node in ipairs(nodes) do
+          local range = vim.treesitter.get_range(node, buf, metadata[capture])
+          local start_row, start_col, end_row, end_col = Range.unpack4(range)
+
+          if hl and end_row >= line and (not is_spell_nav or spell ~= nil) then
+            api.nvim_buf_set_extmark(buf, ns, start_row, start_col, {
+              end_line = end_row,
+              end_col = end_col,
+              hl_group = hl,
+              ephemeral = true,
+              priority = priority,
+              _subpriority = pattern_offset + pattern,
+              conceal = conceal,
+              spell = spell,
+              url = url,
+            })
+          end
+
+          if start_row > line then
+            state.next_row = start_row
+          end
+        end
       end
     end
+
+    pattern_offset = pattern_offset + max_pattern_index
   end)
 end
 

runtime/queries/markdown_inline/highlights.scm

@@ -33,6 +33,11 @@
   ] @markup.link
   (#set! conceal ""))
 
+(inline_link
+  (link_text) @markup.link.label
+  (link_destination) @markup.link
+  (#set! @markup.link.label "url" @markup.link))
+
 ; Conceal image links
 (image
   [

runtime/queries/vimdoc/highlights.scm

@@ -12,21 +12,30 @@
 
 (tag
   "*" @markup.heading.5.marker
-  (#set! conceal "")
-  text: (_) @label)
+  .
+  text: (_) @label
+  .
+  "*" @markup.heading.5.marker
+  (#set! @markup.heading.5.marker conceal ""))
 
 (taglink
-  "|" @markup.link
-  (#set! conceal "")
-  text: (_) @markup.link)
+  "|" @markup.link.delimiter
+  .
+  text: (_) @markup.link
+  .
+  "|" @markup.link.delimiter
+  (#set! @markup.link.delimiter conceal ""))
 
 (optionlink
   text: (_) @markup.link)
 
 (codespan
   "`" @markup.raw.delimiter
-  (#set! conceal "")
-  text: (_) @markup.raw)
+  .
+  text: (_) @markup.raw
+  .
+  "`" @markup.raw.delimiter
+  (#set! @markup.raw.delimiter conceal ""))
 
 ((codeblock) @markup.raw.block
   (#set! "priority" 90))

test/functional/treesitter/highlight_spec.lua

@@ -681,6 +681,12 @@ describe('treesitter highlighting (C)', function()
         ((identifier) @Identifier
          (#set! conceal "")
          (#eq? @Identifier "lstate"))
+
+        ((call_expression
+            function: (identifier) @function
+            arguments: (argument_list) @arguments)
+         (#eq? @function "multiqueue_put")
+         (#set! @function conceal "V"))
       ]]}})
     ]=]
 
@@ -697,7 +703,7 @@ describe('treesitter highlighting (C)', function()
                                                                        |
         LuaRef cb = nlua_ref(, 1);                                     |
                                                                        |
-        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+        {11:V}(main_loop.events, nlua_schedule_event,                       |
                        1, (void *)(ptrdiff_t)cb);                      |
         return 0;                                                      |
       ^}                                                                |
@@ -758,6 +764,44 @@ describe('treesitter highlighting (C)', function()
   end)
 end)
 
+describe('treesitter highlighting (lua)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(65, 18)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { bold = true, foreground = Screen.colors.Blue },
+      [2] = { foreground = Screen.colors.DarkCyan },
+      [3] = { foreground = Screen.colors.Magenta },
+      [4] = { foreground = Screen.colors.SlateBlue },
+      [5] = { bold = true, foreground = Screen.colors.Brown },
+    }
+  end)
+
+  it('supports language injections', function()
+    insert [[
+      local ffi = require('ffi')
+      ffi.cdef("int (*fun)(int, char *);")
+    ]]
+
+    exec_lua [[
+      vim.bo.filetype = 'lua'
+      vim.treesitter.start()
+    ]]
+
+    screen:expect {
+      grid = [[
+        {5:local} {2:ffi} {5:=} {4:require(}{3:'ffi'}{4:)}                                     |
+        {2:ffi}{4:.}{2:cdef}{4:(}{3:"}{4:int}{3: }{4:(}{5:*}{3:fun}{4:)(int,}{3: }{4:char}{3: }{5:*}{4:);}{3:"}{4:)}                           |
+      ^                                                                 |
+      {1:~                                                                }|*14
+                                                                       |
+    ]],
+    }
+  end)
+end)
+
 describe('treesitter highlighting (help)', function()
   local screen
 
@@ -891,3 +935,46 @@ vim.cmd([[
     }
   end)
 end)
+
+describe('treesitter highlighting (markdown)', function()
+  local screen
+
+  before_each(function()
+    screen = Screen.new(40, 6)
+    screen:attach()
+    screen:set_default_attr_ids {
+      [1] = { foreground = Screen.colors.Blue1 },
+      [2] = { bold = true, foreground = Screen.colors.Blue1 },
+      [3] = { bold = true, foreground = Screen.colors.Brown },
+      [4] = { foreground = Screen.colors.Cyan4 },
+      [5] = { foreground = Screen.colors.Magenta1 },
+    }
+  end)
+
+  it('supports hyperlinks', function()
+    local url = 'https://example.com'
+    insert(string.format('[This link text](%s) is a hyperlink.', url))
+    exec_lua([[
+      vim.bo.filetype = 'markdown'
+      vim.treesitter.start()
+    ]])
+
+    screen:expect {
+      grid = [[
+      {4:[}{6:This link text}{4:](}{7:https://example.com}{4:)} is|
+       a hyperlink^.                           |
+      {2:~                                       }|*3
+                                              |
+    ]],
+      attr_ids = {
+        [1] = { foreground = Screen.colors.Blue1 },
+        [2] = { bold = true, foreground = Screen.colors.Blue1 },
+        [3] = { bold = true, foreground = Screen.colors.Brown },
+        [4] = { foreground = Screen.colors.Cyan4 },
+        [5] = { foreground = Screen.colors.Magenta },
+        [6] = { foreground = Screen.colors.Cyan4, url = url },
+        [7] = { underline = true, foreground = Screen.colors.SlateBlue },
+      },
+    }
+  end)
+end)