fix(lsp): refactor text line splitting

Author: dmitmel

runtime/lua/vim/lsp.lua

@@ -115,6 +115,13 @@ local format_line_ending = {
   ["mac"] = '\r',
 }
 
+---@private
+---@param bufnr (number)
+---@returns (string)
+local function buf_get_line_ending(bufnr)
+  return format_line_ending[nvim_buf_get_option(bufnr, 'fileformat')] or '\n'
+end
+
 local client_index = 0
 ---@private
 --- Returns a new, unused client id.
@@ -278,9 +285,10 @@ end
 ---@param bufnr (number) Buffer handle, or 0 for current.
 ---@returns Buffer text as string.
 local function buf_get_full_text(bufnr)
-  local text = table.concat(nvim_buf_get_lines(bufnr, 0, -1, true), '\n')
+  local line_ending = buf_get_line_ending(bufnr)
+  local text = table.concat(nvim_buf_get_lines(bufnr, 0, -1, true), line_ending)
   if nvim_buf_get_option(bufnr, 'eol') then
-    text = text .. '\n'
+    text = text .. line_ending
   end
   return text
 end
@@ -362,9 +370,9 @@ do
     local incremental_changes = function(client)
       local cached_buffers = state_by_client[client.id].buffers
       local curr_lines = nvim_buf_get_lines(bufnr, 0, -1, true)
-      local line_ending = format_line_ending[vim.api.nvim_buf_get_option(0, 'fileformat')]
+      local line_ending = buf_get_line_ending(bufnr)
       local incremental_change = sync.compute_diff(
-        cached_buffers[bufnr], curr_lines, firstline, lastline, new_lastline, client.offset_encoding or 'utf-16', line_ending or '\n')
+        cached_buffers[bufnr], curr_lines, firstline, lastline, new_lastline, client.offset_encoding or 'utf-16', line_ending)
       cached_buffers[bufnr] = curr_lines
       return incremental_change
     end

runtime/lua/vim/lsp/buf.lua

@@ -546,8 +546,7 @@ local function on_code_action_results(results, ctx)
     prompt = 'Code actions:',
     kind = 'codeaction',
     format_item = function(action_tuple)
-      local title = action_tuple[2].title:gsub('\r\n', '\\r\\n')
-      return title:gsub('\n', '\\n')
+      return table.concat(util.split_lines(action_tuple[2].title), '\\n')
     end,
   }, on_user_choice)
 end

runtime/lua/vim/lsp/diagnostic.lua

@@ -102,9 +102,10 @@ local function diagnostic_lsp_to_vim(diagnostics, bufnr, client_id)
       end_lnum = _end.line,
       end_col = line_byte_from_position(buf_lines, _end.line, _end.character, offset_encoding),
       severity = severity_lsp_to_vim(diagnostic.severity),
-      message = diagnostic.message,
+      message = table.concat(vim.lsp.util.split_lines(diagnostic.message), '\n'),
       source = diagnostic.source,
       user_data = {
+        lsp_original_message = diagnostic.message,
         lsp = {
           code = diagnostic.code,
           codeDescription = diagnostic.codeDescription,
@@ -132,7 +133,7 @@ local function diagnostic_vim_to_lsp(diagnostics)
         },
       },
       severity = severity_vim_to_lsp(diagnostic.severity),
-      message = diagnostic.message,
+      message = (diagnostic.user_data and diagnostic.user_data.lsp_original_message) or diagnostic.message,
       source = diagnostic.source,
     }, diagnostic.user_data and (diagnostic.user_data.lsp or {}) or {})
   end, diagnostics)

runtime/lua/vim/lsp/handlers.lua

@@ -82,8 +82,7 @@ M['window/showMessageRequest'] = function(_, result)
   print(result.message)
   local option_strings = {result.message, "\nRequest Actions:"}
   for i, action in ipairs(actions) do
-    local title = action.title:gsub('\r\n', '\\r\\n')
-    title = title:gsub('\n', '\\n')
+    local title = table.concat(util.split_lines(action.title), '\\n')
     table.insert(option_strings, string.format("%d. %s", i, title))
   end
 

runtime/lua/vim/lsp/util.lua

@@ -85,9 +85,95 @@ local function get_border_size(opts)
   return { height = height, width = width }
 end
 
----@private
-local function split_lines(value)
-  return split(value, '\n', true)
+--- Splits the given text into lines according to LSP's definition of text
+--- lines. Quoting the specification: "To ensure that both client and server
+--- split the string into the same line representation the protocol specifies
+--- the following end-of-line sequences: '\n', '\r\n' and '\r'."
+---
+---@see |vim.lsp.util.split_lines_iter()|
+---@see https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocuments
+function M.split_lines(text, opts)
+  local lines = {}
+  local i = 1
+  for _, line in M.split_lines_iter(text, opts) do
+    lines[i] = line
+    i = i + 1
+  end
+  return lines
+end
+
+--- Same as |vim.lsp.util.split_lines()|, but returns an iterator instead of a
+--- list of lines.
+function M.split_lines_iter(text, opts)
+  validate {
+    text = { text, 's' };
+    opts = { opts, 't', true };
+  }
+  opts = opts or {}
+  validate {
+    ['opts.keep_line_endings'] = { opts.keep_line_endings, 'b', true };
+    ['opts.keep_final_eol'] = { opts.keep_final_eol, 'b', true };
+  }
+  local keep_line_endings = opts.keep_line_endings
+  local keep_final_eol = opts.keep_final_eol
+
+  local string_find = string.find
+  local string_sub = string.sub
+  local math_min = math.min
+
+  -- Most text will be LF-separated anyway, so needless \r lookups which would
+  -- have to scan the whole string can be skipped. There is no such flag for \n
+  -- because in the CRLF case the \n lookup still must be performed, and let's
+  -- face it - nobody uses CR linebreaks anymore.
+  local no_carrige_returns_left = false
+
+  -- NOTE: string.find is implemented using memchr in LuaJIT, which will most
+  -- likely be faster than running our own loops.
+  local function find_curr_line_ending(start_idx)
+    local lf_idx = string_find(text, '\n', start_idx, true)
+    local cr_idx = nil
+    if not no_carrige_returns_left then
+      cr_idx = string_find(text, '\r', start_idx, true)
+      no_carrige_returns_left = not cr_idx
+    end
+    if lf_idx and cr_idx then
+      if cr_idx + 1 == lf_idx then
+        return cr_idx, 2
+      else
+        return math_min(lf_idx, cr_idx), 1
+      end
+    else
+      return lf_idx or cr_idx, 1
+    end
+  end
+
+  local line_start_idx = 1
+  local done = false
+  return function()
+    if done then
+      return
+    end
+
+    local line_ending_idx, line_ending_len = find_curr_line_ending(line_start_idx)
+    if not line_ending_idx then
+      done = true
+      local line = string_sub(text, line_start_idx)
+      return line_start_idx, line
+    end
+
+    local line_end_idx = line_ending_idx - 1
+    if keep_line_endings then
+      line_end_idx = line_end_idx + line_ending_len
+    end
+    local line = string_sub(text, line_start_idx, line_end_idx)
+
+    local prev_line_start_idx = line_start_idx
+    line_start_idx = line_ending_idx + line_ending_len
+    if line_start_idx > #text and not keep_final_eol then
+      done = true
+    end
+    return prev_line_start_idx, line
+  end
 end
 
 --- Replaces text in a range with new text.
@@ -324,7 +410,7 @@ function M.apply_text_edits(text_edits, bufnr)
       start_col = get_line_byte_from_position(bufnr, text_edit.range.start),
       end_row = text_edit.range['end'].line,
       end_col  = get_line_byte_from_position(bufnr, text_edit.range['end']),
-      text = vim.split(text_edit.newText, '\n', true),
+      text = M.split_lines(text_edit.newText, { keep_final_eol = true }),
     }
     vim.api.nvim_buf_set_text(bufnr, e.start_row, e.start_col, e.end_row, e.end_col, e.text)
 
@@ -647,7 +733,7 @@ function M.convert_input_to_markdown_lines(input, contents)
   contents = contents or {}
   -- MarkedString variation 1
   if type(input) == 'string' then
-    list_extend(contents, split_lines(input))
+    list_extend(contents, M.split_lines(input))
   else
     assert(type(input) == 'table', "Expected a table for Hover.contents")
     -- MarkupContent
@@ -665,13 +751,13 @@ function M.convert_input_to_markdown_lines(input, contents)
       end
 
       -- assert(type(value) == 'string')
-      list_extend(contents, split_lines(value))
+      list_extend(contents, M.split_lines(value))
     -- MarkupString variation 2
     elseif input.language then
       -- Some servers send input.value as empty, so let's ignore this :(
       -- assert(type(input.value) == 'string')
       table.insert(contents, "```"..input.language)
-      list_extend(contents, split_lines(input.value or ''))
+      list_extend(contents, M.split_lines(input.value or ''))
       table.insert(contents, "```")
     -- By deduction, this must be MarkedString[]
     else
@@ -718,7 +804,7 @@ function M.convert_signature_help_to_markdown_lines(signature_help, ft, triggers
     -- wrap inside a code block so stylize_markdown can render it properly
     label = ("```%s\n%s\n```"):format(ft, label)
   end
-  vim.list_extend(contents, vim.split(label, '\n', true))
+  vim.list_extend(contents, M.split_lines(label))
   if signature.documentation then
     M.convert_input_to_markdown_lines(signature.documentation, contents)
   end

test/functional/plugin/lsp_spec.lua

@@ -2578,4 +2578,91 @@ describe('LSP', function()
       }
     end)
   end)
+
+  describe('lsp.util.split_lines', function()
+    local function test_split_lines(expected_lines, ...)
+      eq(expected_lines, exec_lua('return vim.lsp.util.split_lines(...)', ...))
+    end
+
+    it('handles empty strings', function()
+      test_split_lines({''}, '')
+    end)
+
+    it('handles strings without line ending characters', function()
+      test_split_lines({'hi!'}, 'hi!')
+    end)
+
+    it('handles strings without a final EOL', function()
+      test_split_lines({'abcdef', 'ghijkl'}, 'abcdef\nghijkl')
+      test_split_lines({'a', 'b', 'd'}, 'a\r\nb\nd')
+    end)
+
+    local test_text = table.concat({
+      '#include <stdio.h>\n',
+      'int main() {\r\n',
+      '  printf(\r"привет мир");\r\n\r',  -- Let's throw in some multibyte because why not
+      '  return 0;\n\r',
+      '}\n',
+    })
+
+    it('handles text with mixed newlines', function()
+      test_split_lines({
+        '#include <stdio.h>',
+        'int main() {',
+        '  printf(',
+        '"привет мир");',
+        '',
+        '  return 0;',
+        '',
+        '}',
+      }, test_text)
+    end)
+
+    it('handles keep_line_endings', function()
+      test_split_lines({
+        '#include <stdio.h>\n',
+        'int main() {\r\n',
+        '  printf(\r',
+        '"привет мир");\r\n',
+        '\r',
+        '  return 0;\n',
+        '\r',
+        '}\n',
+      }, test_text, { keep_line_endings = true })
+    end)
+
+    it('handles keep_final_eol', function()
+      test_split_lines({ 'абв',   'где',   '' }, 'абв\nгде\n', { keep_final_eol = true                           })
+      test_split_lines({ 'абв',   'где'       }, 'абв\nгде',   { keep_final_eol = true                           })
+      test_split_lines({ 'абв\n', 'где\n', '' }, 'абв\nгде\n', { keep_final_eol = true, keep_line_endings = true })
+      test_split_lines({ 'абв\n', 'где'       }, 'абв\nгде',   { keep_final_eol = true, keep_line_endings = true })
+    end)
+
+    it('handles a final CRLF', function()
+      test_split_lines({ 'test' }, 'test\r\n')
+    end)
+
+    it('handles the no_carrige_returns_left optimization correctly', function()
+      test_split_lines({ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h' }, 'a\r\nb\r\nc\rd\ne\nf\ng\nh\n')
+    end)
+
+    it('the iterator returns correct line start indexes', function()
+      eq({
+        { 1, '#include <stdio.h>' },
+        { 20, 'int main() {' },
+        { 34, '  printf(' },
+        { 44, '"привет мир");' },
+        { 69, '' },
+        { 70, '  return 0;' },
+        { 82, '' },
+        { 83, '}' },
+      }, exec_lua([[
+        local results = {}
+        for start_idx, line in vim.lsp.util.split_lines_iter(...) do
+          table.insert(results, { start_idx, line })
+        end
+        return results
+      ]], test_text))
+    end)
+  end)
 end)