decor: sketch new decorations API

return decorations back

lol no nvim_buf_get_virtual_text

share decorations that are hl only to avoid alloc avalanche

src/nvim/extmark.c

@@ -43,6 +43,13 @@
 # include "extmark.c.generated.h"
 #endif
 
+static PMap(uint64_t) *hl_decors;
+
+void extmark_init(void)
+{
+  hl_decors = pmap_new(uint64_t)();
+}
+
 static ExtmarkNs *buf_ns_ref(buf_T *buf, uint64_t ns_id, bool put) {
   if (!buf->b_extmark_ns) {
     if (!put) {
@@ -291,31 +298,44 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
 // will be searched to the start, or end
 // dir can be set to control the order of the array
 // amount = amount of marks to find or -1 for all
-ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
-                         int l_row, colnr_T l_col,
-                         int u_row, colnr_T u_col,
-                         int64_t amount, bool reverse)
+ExtmarkInfoArray extmark_get(buf_T *buf, uint64_t ns_id,
+                             int l_row, colnr_T l_col,
+                             int u_row, colnr_T u_col,
+                             int64_t amount, bool reverse)
 {
-  ExtmarkArray array = KV_INITIAL_VALUE;
-  MarkTreeIter itr[1] = { 0 };
+  ExtmarkInfoArray array = KV_INITIAL_VALUE;
+  MarkTreeIter itr[1];
   // Find all the marks
   marktree_itr_get_ext(buf->b_marktree, (mtpos_t){ l_row, l_col },
                        itr, reverse, false, NULL);
   int order = reverse ? -1 : 1;
   while ((int64_t)kv_size(array) < amount) {
     mtmark_t mark = marktree_itr_current(itr);
+    mtpos_t endpos = { -1, -1 };
     if (mark.row < 0
         || (mark.row - u_row) * order > 0
         || (mark.row == u_row && (mark.col - u_col) * order > 0)) {
       break;
     }
+    if (mark.id & MARKTREE_END_FLAG) {
+      goto next_mark;
+    } else if (mark.id & MARKTREE_PAIRED_FLAG) {
+      endpos = marktree_lookup(buf->b_marktree, mark.id | MARKTREE_END_FLAG,
+                               NULL);
+    }
+
+
     ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                       mark.id);
     if (item.ns_id == ns_id) {
       kv_push(array, ((ExtmarkInfo) { .ns_id = item.ns_id,
                                       .mark_id = item.mark_id,
-                                      .row = mark.row, .col = mark.col }));
+                                      .row = mark.row, .col = mark.col,
+                                      .end_row = endpos.row,
+                                      .end_col = endpos.col,
+                                      .decor = item.decor }));
     }
+next_mark:
     if (reverse) {
       marktree_itr_prev(buf->b_marktree, itr);
     } else {
@@ -329,7 +349,7 @@ ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
 ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
 {
   ExtmarkNs *ns = buf_ns_ref(buf, ns_id, false);
-  ExtmarkInfo ret = { 0, 0, -1, -1 };
+  ExtmarkInfo ret = { 0, 0, -1, -1, -1, -1, NULL };
   if (!ns) {
     return ret;
   }
@@ -340,12 +360,22 @@ ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
   }
 
   mtpos_t pos = marktree_lookup(buf->b_marktree, mark, NULL);
+  mtpos_t endpos = { -1, -1 };
+  if (mark & MARKTREE_PAIRED_FLAG) {
+    endpos = marktree_lookup(buf->b_marktree, mark | MARKTREE_END_FLAG, NULL);
+  }
   assert(pos.row >= 0);
 
+  ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
+                                                    mark);
+
   ret.ns_id = ns_id;
   ret.mark_id = id;
   ret.row = pos.row;
   ret.col = pos.col;
+  ret.end_row = endpos.row;
+  ret.end_col = endpos.col;
+  ret.decor = item.decor;
 
   return ret;
 }
@@ -682,6 +712,7 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
 {
   colnr_T hl_start = 0;
   colnr_T hl_end = 0;
+  Decoration *decor = decoration_hl(hl_id);
 
   // TODO(bfredl): if decoration had blocky mode, we could avoid this loop
   for (linenr_T lnum = pos_start.lnum; lnum <= pos_end.lnum; lnum ++) {
@@ -706,14 +737,28 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
       hl_start = pos_start.col + offset;
       hl_end = pos_end.col + offset;
     }
-    Decoration *decor = xcalloc(1, sizeof(*decor));
-    decor->hl_id = hl_id;
     (void)extmark_set(buf, (uint64_t)src_id, 0,
                       (int)lnum-1, hl_start, (int)lnum-1+end_off, hl_end,
                       decor, kExtmarkUndo);
   }
 }
 
+Decoration *decoration_hl(int hl_id)
+{
+  assert(hl_id > 0);
+  Decoration **dp = (Decoration **)pmap_ref(uint64_t)(hl_decors,
+                                                      (uint64_t)hl_id, true);
+  if (*dp) {
+    return *dp;
+  }
+
+  Decoration *decor = xcalloc(1, sizeof(*decor));
+  decor->hl_id = hl_id;
+  decor->shared = true;
+  *dp = decor;
+  return decor;
+}
+
 void decoration_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
 {
   if (decor->hl_id && row2 >= row1) {
@@ -727,7 +772,7 @@ void decoration_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
 
 void free_decoration(Decoration *decor)
 {
-  if (decor) {
+  if (decor && !decor->shared) {
     clear_virttext(&decor->virt_text);
     xfree(decor);
   }

src/nvim/extmark.h

@@ -13,9 +13,12 @@ typedef struct
   uint64_t mark_id;
   int row;
   colnr_T col;
+  int end_row;
+  colnr_T end_col;
+  Decoration *decor;
 } ExtmarkInfo;
 
-typedef kvec_t(ExtmarkInfo) ExtmarkArray;
+typedef kvec_t(ExtmarkInfo) ExtmarkInfoArray;
 
 
 // delete the columns between mincol and endcol

src/nvim/extmark_defs.h

@@ -17,6 +17,7 @@ typedef struct
   int hl_id;  // highlight group
   VirtText virt_text;
   // TODO(bfredl): style, signs, etc
+  bool shared;  // shared decoration, don't free
 } Decoration;
 
 typedef struct

src/nvim/main.c

@@ -21,6 +21,7 @@
 #include "nvim/ex_cmds.h"
 #include "nvim/ex_cmds2.h"
 #include "nvim/ex_docmd.h"
+#include "nvim/extmark.h"
 #include "nvim/fileio.h"
 #include "nvim/fold.h"
 #include "nvim/getchar.h"
@@ -160,6 +161,7 @@ void early_init(mparm_T *paramp)
   env_init();
   fs_init();
   handle_init();
+  extmark_init();
   eval_init();          // init global variables
   init_path(argv0 ? argv0 : "nvim");
   init_normal_cmds();   // Init the table of Normal mode commands.

src/nvim/map.h

@@ -73,6 +73,7 @@ MAP_DECLS(String, handle_T)
 #define pmap_has(T) map_has(T, ptr_t)
 #define pmap_key(T) map_key(T, ptr_t)
 #define pmap_put(T) map_put(T, ptr_t)
+#define pmap_ref(T) map_ref(T, ptr_t)
 /// @see pmap_del2
 #define pmap_del(T) map_del(T, ptr_t)
 #define pmap_clear(T) map_clear(T, ptr_t)

test/functional/api/extmark_spec.lua

@@ -18,13 +18,13 @@ local function expect(contents)
 end
 
 local function check_undo_redo(ns, mark, sr, sc, er, ec) --s = start, e = end
-  local rv = curbufmeths.get_extmark_by_id(ns, mark)
+  local rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
   feed("u")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({sr, sc}, rv)
   feed("<c-r>")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
 end
 
@@ -42,7 +42,7 @@ local function get_extmarks(ns_id, start, end_, opts)
   if opts == nil then
     opts = {}
   end
-  return curbufmeths.get_extmarks(ns_id, start, end_, opts)
+  return curbufmeths.get_extmarks(ns_id, start, end_, opts, false)
 end
 
 local function batch_set(ns_id, positions)
@@ -96,7 +96,7 @@ describe('API/extmarks', function()
   it('adds, updates  and deletes marks', function()
     local rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({positions[1][1], positions[1][2]}, rv)
     -- Test adding a second mark on same row works
     rv = set_extmark(ns, marks[2], positions[2][1], positions[2][2])
@@ -105,14 +105,14 @@ describe('API/extmarks', function()
     -- Test an update, (same pos)
     rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({positions[2][1], positions[2][2]}, rv)
     -- Test an update, (new pos)
     row = positions[1][1]
     col = positions[1][2] + 1
     rv = set_extmark(ns, marks[1], row, col)
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({row, col}, rv)
 
     -- remove the test marks
@@ -435,7 +435,7 @@ describe('API/extmarks', function()
       ~              |
                      |
     ]])
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 6}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 6)
   end)
@@ -909,9 +909,9 @@ describe('API/extmarks', function()
     -- Set the mark before the cursor, should stay there
     set_extmark(ns, marks[2], 0, 10)
     feed("i<cr><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({0, 10}, rv)
     check_undo_redo(ns, marks[1], 0, 12, 1, 3)
   end)
@@ -924,12 +924,12 @@ describe('API/extmarks', function()
     feed("0iint <esc>A {<cr><esc>0i1M1<esc>")
     set_extmark(ns, marks[1], 1, 1)
     feed("0i<c-f><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
     check_undo_redo(ns, marks[1], 1, 1, 1, 3)
     -- now check when cursor at eol
     feed("uA<c-f><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
   end)
 
@@ -940,12 +940,12 @@ describe('API/extmarks', function()
     feed("0i<tab><esc>")
     set_extmark(ns, marks[1], 0, 3)
     feed("bi<c-d><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 1)
     -- check when cursor at eol
     feed("uA<c-d><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
   end)
 
@@ -1075,7 +1075,7 @@ describe('API/extmarks', function()
     check_undo_redo(ns, marks[5], 2, 0, 3, 0)
     feed('u')
     feed([[:1,2s:3:\rxx<cr>]])
-    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3]))
+    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3], false))
   end)
 
   it('substitions over multiple lines with replace in substition', function()
@@ -1314,16 +1314,16 @@ describe('API/extmarks', function()
     eq("Invalid ns_id", pcall_err(set_extmark, ns_invalid, marks[1], positions[1][1], positions[1][2]))
     eq("Invalid ns_id", pcall_err(curbufmeths.del_extmark, ns_invalid, marks[1]))
     eq("Invalid ns_id", pcall_err(get_extmarks, ns_invalid, positions[1], positions[2]))
-    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1]))
+    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1], false))
   end)
 
   it('when col = line-length, set the mark on eol', function()
     set_extmark(ns, marks[1], 0, -1)
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
     -- Test another
     set_extmark(ns, marks[1], 0, -1)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
   end)
 
@@ -1336,7 +1336,7 @@ describe('API/extmarks', function()
     local invalid_col = init_text:len() + 1
     local invalid_lnum = 3
     eq('line value outside range', pcall_err(set_extmark, ns, marks[1], invalid_lnum, invalid_col))
-    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1]))
+    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1], false))
   end)
 
   it('bug from check_col in extmark_set', function()
@@ -1361,7 +1361,7 @@ describe('API/extmarks', function()
     local buf = request('nvim_create_buf', 0, 1)
     request('nvim_buf_set_lines', buf, 0, -1, 1, {"", ""})
     local id = bufmeths.set_extmark(buf, ns, 1, 0, {})
-    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}))
+    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}, false))
   end)
 
   it('does not crash with append/delete/undo seqence', function()

test/functional/ui/bufhl_spec.lua

@@ -690,7 +690,7 @@ describe('Buffer highlighting', function()
     end)
 
     it('can be retrieved', function()
-      local get_virtual_text = curbufmeths.get_virtual_text
+      local get_extmarks = curbufmeths.get_extmarks
       local line_count = curbufmeths.line_count
 
       local s1 = {{'Köttbullar', 'Comment'}, {'Kräuterbutter'}}
@@ -699,12 +699,14 @@ describe('Buffer highlighting', function()
       -- TODO: only a virtual text from the same ns curretly overrides
       -- an existing virtual text. We might add a prioritation system.
       set_virtual_text(id1, 0, s1, {})
-      eq(s1, get_virtual_text(0))
+      eq({{1, 0, 0, {virt_text = s1}}}, get_extmarks(id1, {0,0}, {0, -1}, {}, true))
 
-      set_virtual_text(-1, line_count(), s2, {})
-      eq(s2, get_virtual_text(line_count()))
+      -- TODO: is this really valid? shouldn't the max be line_count()-1?
+      local lastline = line_count()
+      set_virtual_text(id1, line_count(), s2, {})
+      eq({{3, lastline, 0, {virt_text = s2}}}, get_extmarks(id1, {lastline,0}, {lastline, -1}, {}, true))
 
-      eq({}, get_virtual_text(line_count() + 9000))
+      eq({}, get_extmarks(id1, {lastline+9000,0}, {lastline+9000, -1}, {}, false))
     end)
 
     it('is not highlighted by visual selection', function()