New Collector that is configurable per filetype

[edlandm]

I'm loving this plugin and how configurable it is. I especially like the treesitter integration, but I found myself wanting a way to provide a fallback for filetypes that don't have treesitter parsers.
Simple example: The fugitive.vim gitcommit filetype. I wanted to be quickly yank my modified filepaths:

I made this custom Collector that allows me to specify patterns as well as override the treesitter metadata for a given filetype. In practice I have a central "quick-yank" mapping that targets all of the most likely patterns without needing a different mapping definition for each filetype (ie defining a mapping in each ftplugin file).
It also allows me to define custom treesitter queries for filetypes like sql (specifically I'm using a plsql parser d that don't have a lot of the most common node types.

Here's a minimal config with my implementation. Stylistically it's likely not exactly how you would do it, but I wanted to introduce the concept in case its something you think is worth adding to the plugin.

return {
  "FluxxField/smart-motion.nvim",
  event = 'VeryLazy',
  opts = {
    -- ... nothing special in here for this demo
  },
  config = function(_, opts)
    local sm = require 'smart-motion'
    sm.setup(opts)

    -- allow filetype overrides for ts_node_types or custom function (coroutine) to
    -- pattern match (eg gitcommit doesn't have treesitter so I pattern-match
    -- to find the listed files)
    local FileTypeTSCollector = {
      name = 'ft_treesitter_or_patterns',
      ft = {
        -- supplying a string pattern or a list of patterns will find matches
        -- via regex (ie lua pattern matching)
        -- NOTE: need to include `()` to denote the start and end of the target
        gitcommit = 'modified:%s+()(%S.*)()$',
        -- can specify multiple patterns as {patterns:string[]}
        --[[
        gitcommit = {
          patterns = {
            'modified:%s+()(%S.*)()$',
            'On branch ()(%S+)()$',
          }
        }
        --]]
        -- you can also override any of the treesitter metadata; here's
        -- a custom query for (pl)sql
        sql = {
          ts_query = [[
            (literal_string) @string
            (expression
              (referenced_element) @element)
            (select_list) @slist
            (select_list_element
              (identifier) @alias)
            (into_clause
              (referenced_element_repeat
                (referenced_element) @into_ref))
            (table_list
              (table_list_element [
                (referenced_element
                  ref_name: (identifier) @table_name)
                (join_clause
                  (referenced_element) @join_table_name)
                ]))
            (statement
              (referenced_element
                ref_name_parent: (identifier) @var_name))
            (function_definition
              fnc_name: (identifier) @fn_name)
          ]]
        },
      },
      default_treesitter = {
        ts_node_types = {
          'hash_key_symbol',
          'string_content',
          'constant',
          'scope_resolution',
          'identifier',
          'inline',
        },
      },
      yield_matching_lines = function(pattern) -- (string|table) -> coroutine
        local getlines = function(bufnr)
          return ipairs(vim.api.nvim_buf_get_lines(bufnr, 0, -1, true))
        end
        local yield_match = function(line_number, line, p)
          local start_pos, match, end_pos = line:match(p)
          if match then
            coroutine.yield({
              text = match,
              line_number = line_number-1,
              start_pos = { row = line_number-1, col = start_pos-1 },
              end_pos = { row = line_number-1, col = end_pos },
              type = 'words',
            })
          end
        end
        local fn = function(ctx) -- match one pattern
          for line_number, line in getlines(ctx.bufnr) do
            yield_match(line_number, line, pattern)
          end
        end
        if type(pattern) == 'table' then
          fn = function(ctx) -- match multiple patterns
            for line_number, line in getlines(ctx.bufnr) do
              for _, p in ipairs(pattern) do
                yield_match(line_number, line, p)
              end
            end
          end
        end
        return coroutine.create(fn)
      end
    }
    FileTypeTSCollector.__index = FileTypeTSCollector

    function FileTypeTSCollector.run()
      return coroutine.create(function(ctx, cfg, motion_state)
        local filetype = vim.bo[ctx.bufnr].filetype
        local extractor = FileTypeTSCollector.ft[filetype]
        local co, state = nil, motion_state

        if extractor then
          if type(extractor) == 'table' then
            if extractor.patterns then
              co = FileTypeTSCollector.yield_matching_lines(extractor.patterns)
            else
              state = extractor
            end
          elseif type(extractor) == 'string' then
            co = FileTypeTSCollector.yield_matching_lines(extractor)
          elseif type(extractor) == 'function' then
            co = extractor()
          else
            error('invalid extractor type, got: '..type(extractor))
          end
        end

        if not co then
          co = sm.collectors.get_by_name('treesitter').run()
        end
        assert(co, 'could not get extractor coroutine')

        local ok, yield = true, nil
        while ok do
          ok, yield = coroutine.resume(co, ctx, cfg, state)
          if ok then
            coroutine.yield(yield)
          elseif yield then
            vim.print('Error running coroutine: ' .. yield)
          end
        end
      end)
    end

    sm.collectors.register('ft_targets_or_treesitter', FileTypeTSCollector)

    sm.motions.register('ry', {
      collector = 'ft_targets_or_treesitter',
      extractor = 'pass_through',
      filter = 'filter_visible',
      visualizer = 'hint_before',
      modifier = 'weight_distance',
      action = 'remote_yank',
      map = true,
      trigger_key = 'y_',
      modes = { 'n' },
      metadata = {
        label = 'Remote Yank',
        motion_state = FileTypeTSCollector.default_treesitter,
      },
    })
  end,
}

Cheers!

[FluxxField]

Hey @edlandm, thanks for the detailed writeup and working implementation! Really cool to see someone leveraging the collector/extractor API like this.

The use case is solid, filetypes without treesitter parsers (like fugitive's gitcommit) are a real gap right now, and having a single keymap that adapts per-filetype instead of scattering mappings across ftplugin files is a much better UX.

I like the layered approach:

1. Filetype-specific pattern matching for non-TS filetypes
2. Per-filetype treesitter overrides (custom queries, node types)
3. Fallback to default treesitter config

This is something I'd be interested in adding as a built-in collector. The core idea fits naturally into the existing architecture. It's essentially a filetype-aware wrapper around the treesitter collector with a pattern-matching escape hatch.

A few things I'd want to think through for a built-in version:

• How the filetype config would integrate with setup() opts (probably under a filetype_targets key or similar)
• Whether pattern targets should go through the normal extractor pipeline or stay self-contained like your implementation
• Making sure the pattern format is intuitive for users who aren't Lua pattern experts

Thanks for putting this together! It's a great proof of concept and exactly the kind of feedback that helps shape the plugin. I'll keep this open as I think through the design. I will start playing around and thinking about this today and comment some more on what I add and explore.

Thanks again

[FluxxField]

@edlandm

Update: This is now built into the plugin on the dev branch!

Your writeup and working proof of concept directly shaped the design. I want to walk through what I built, why I made the choices I did, and where this can go.

The Problem You Identified

Your core insight was that smart-motion's pipeline is powerful, but it assumes treesitter is always available. That's not true for:

• Fugitive's gitcommit filetype (no TS parser)
• Filetypes with parsers that lack common node types (your SQL/PLSQL example)
• Log files, CSV, config formats, and other structured text
• Any buffer where regex is simply the better tool

Your solution bundled two things into one collector: pattern matching (the regex-based collection) and filetype routing (choosing the right strategy per filetype). I split these into two independent modules because each solves a distinct problem and is useful on its own.

What I Built

1. The patterns Collector

A new collector module (collectors/patterns.lua) that finds targets using vim regex patterns against buffer lines.

What it does:

• Reads motion_state.patterns an array of vim regex strings
• Iterates buffer lines within a window around the cursor (same windowing as the lines collector)
• Runs vim.fn.matchstrpos() for each pattern on each line, yielding standard targets
• Supports multiple matches per line (advances past each match)
• Optional patterns_whole_line = true mode where the entire matching line becomes the target

Why vim regex instead of Lua patterns:

Your implementation used Lua patterns with () capture groups for position marking, which is clever but niche. I went with vim regex because:

• matchstrpos() is already used throughout the plugin (native search, char repeat, multi-edit, visual select, treesitter search)
• Neovim users already know vim regex - \v very-magic mode, \zs/\ze for match boundaries, etc.
• No need for manual () position captures - matchstrpos gives us start/end positions directly

Why it yields standard targets:

Your implementation was self-contained. The collector yielded final targets and used pass_through as the extractor. That works, but it locks the pattern matching inside one collector. By yielding the same target format as every other collector
(text, start_pos, end_pos, type, metadata), the patterns collector composes with the entire pipeline:

• Any filter works: filter_visible, directional filters, filter_words_after_cursor
• Any visualizer works: hint_start, hint_before, hint_end
• Any action works: jump, jump_centered, remote_yank, operators via infer
• Multi-window support works automatically via create_multi_window_collector

Standalone usage example (no filetype dispatch needed):

-- Jump to file paths anywhere in the buffer
sm.motions.register("gf", {
  collector = "patterns",
  extractor = "pass_through",
  filter = "filter_visible",
  visualizer = "hint_start",
  action = "jump",
  map = true,
  trigger_key = "gf",
  modes = { "n" },
  metadata = {
    motion_state = {
      patterns = { "\\v\\f+" },
    },
  },
})

2. The Filetype Dispatch Middleware

A pre-pipeline middleware (core/engine/filetype_dispatch.lua) that swaps pipeline modules based on the current buffer's filetype.

What it does:

• Runs in setup.run() after the motion is resolved from the registry but before modules are loaded
• Checks motion_state.filetype_overrides[current_filetype]
• If a match exists, deep-merges the override into the motion definition
• Can swap any pipeline module: collector, extractor, modifier, filter, visualizer, action
• Can override any motion_state metadata: patterns, ts_query, ts_node_types, etc.

Why middleware instead of a meta-collector:

Your implementation was a collector that wrapped other collectors. It checked the filetype internally and delegated to either your pattern matcher or the treesitter collector. This works for the collector stage, but it can't swap extractors, filters,
or visualizers per filetype.

The middleware approach is more powerful because it operates before the pipeline runs. It modifies the motion definition itself, so the entire pipeline sees the overridden config. This means:

• A gitcommit override can swap the collector to patterns AND the filter to filter_visible AND the visualizer to hint_before, all in one override
• An sql override can keep the treesitter collector but swap in a custom ts_query, without changing anything else
• The override cascades to infer, so d]] in a gitcommit buffer automatically uses the pattern-based pipeline

Why per-motion instead of global config:

I considered a global filetype_overrides in setup() opts, but per-motion scoping is more flexible. Your treesitter jump (]]) might need different overrides than your remote yank. Each motion controls its own filetype behavior:

-- This motion uses patterns for gitcommit, custom TS query for SQL
["]]"] = {
  collector = "treesitter",
  extractor = "pass_through",
  filter = "filter_visible",
  visualizer = "hint_start",
  action = "jump_centered",
  metadata = {
    motion_state = {
      ts_node_types = { "function_declaration", "function_definition" },
      filetype_overrides = {
        gitcommit = {
          collector = "patterns",
          motion_state = {
            patterns = { "\\vmodified:\\s+\\zs\\S.*$" },
          },
        },
        sql = {
          motion_state = {
            ts_query = [[(function_definition name: (identifier) @fn)]],
          },
        },
      },
    },
  },
}

Zero cost when unused:

The middleware has multiple early-return guards. If a motion has no filetype_overrides configured (which is the case for every existing motion), the function returns immediately before doing any work. No performance impact on existing motions.

Your Fugitive Use Case

Your original example: quickly yanking modified filepaths from gitcommit works like this:

sm.motions.register("ry", {
  collector = "treesitter",
  extractor = "pass_through",
  filter = "filter_visible",
  visualizer = "hint_before",
  modifier = "weight_distance",
  action = "remote_yank",
  map = true,
  trigger_key = "ry",
  modes = { "n" },
  metadata = {
    label = "Remote Yank",
    motion_state = {
      ts_node_types = { "identifier", "string_content" },
      multi_window = true,
      filetype_overrides = {
        gitcommit = {
          collector = "patterns",
          motion_state = {
            patterns = {
              "\\vmodified:\\s+\\zs\\S.*$",
              "\\vnew file:\\s+\\zs\\S.*$",
              "\\vdeleted:\\s+\\zs\\S.*$",
            },
          },
        },
      },
    },
  },
})

One keymap (ry), works everywhere:

• In code files → treesitter-based remote yank (identifiers, strings)
• In gitcommit buffers → pattern-based remote yank (file paths)
• Operators compose automatically → dry would delete-yank to a target

Other Applications

Beyond your fugitive use case, here are some things this enables:

Log files - Jump to timestamps, error levels, or IP addresses:

filetype_overrides = {
  log = {
    collector = "patterns",
    motion_state = {
      patterns = {
        "\\v\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}",  -- ISO timestamps
        "\\v(ERROR|WARN|FATAL)",                            -- severity levels
        "\\v\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}",   -- IP addresses
      },
    },
  },
}

Markdown without TS - Jump to headers, links, or code fences:

filetype_overrides = {
  markdown = {
    collector = "patterns",
    motion_state = {
      patterns = { "\\v^#{1,6}\\s+\\zs.*$" },  -- headers
    },
  },
}

Per-language TS customization - Different node types or queries per language without separate motions:

filetype_overrides = {
  python = {
    motion_state = {
      ts_node_types = { "function_definition", "class_definition", "decorated_definition" },
    },
  },
  go = {
    motion_state = {
      ts_node_types = { "function_declaration", "method_declaration", "type_declaration" },
    },
  },
}

Mixed strategies, some filetypes use treesitter, others use patterns, all from one keymap. The dispatch handles the routing transparently.

Your proof of concept was the catalyst for this. The layered fallback idea (filetype-specific → default treesitter) is preserved exactly. We just separated the layers into independent, composable modules.

I would love to hear if this covers your use cases and if you have ideas for other applications!

[FluxxField]

Deployed with this PR #132

[edlandm]

Hey this is awesome! This is exactly why I wanted to bring it to you and leverage your overall vision for the plugin rather than just letting the idea stay with my hack.
Great idea to use vim regex over lua patterns. It provides the added benefit of being able to use vim's own search to refine your pattern rather than lua.
I also think it's great that you took the filetype-aware component and opted for a middleware so that it can be applied to not only collectors.
Thanks for taking my idea and turning it into something even more helpful than my original use case!