[ruby/reline] Completely support full-width characters in

differential rendering
(ruby/reline#654)

* Add a cut variation of Reline::Unicode.take_range method take_mbchar_range

* Consider fullwidth take_range in differential rendering

ruby/reline@29714df09f

lib/reline/line_editor.rb

@@ -414,8 +414,13 @@ def render_line_differential(old_items, new_items)
         @output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}"
       else
         x, w, content = new_items[level]
-        content = Reline::Unicode.take_range(content, base_x - x, width) unless x == base_x && w == width
-        Reline::IOGate.move_cursor_column base_x
+        cover_begin = base_x != 0 && new_levels[base_x - 1] == level
+        cover_end = new_levels[base_x + width] == level
+        pos = 0
+        unless x == base_x && w == width
+          content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true)
+        end
+        Reline::IOGate.move_cursor_column x + pos
         @output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}"
       end
       base_x += width
@@ -699,13 +704,6 @@ def add_dialog_proc(name, p, context = nil)
 
   DIALOG_DEFAULT_HEIGHT = 20
 
-  private def padding_space_with_escape_sequences(str, width)
-    padding_width = width - calculate_width(str, true)
-    # padding_width should be only positive value. But macOS and Alacritty returns negative value.
-    padding_width = 0 if padding_width < 0
-    str + (' ' * padding_width)
-  end
-
   private def dialog_range(dialog, dialog_y)
     x_range = dialog.column...dialog.column + dialog.width
     y_range = dialog_y + dialog.vertical_offset...dialog_y + dialog.vertical_offset + dialog.contents.size
@@ -778,7 +776,7 @@ def add_dialog_proc(name, p, context = nil)
     dialog.contents = contents.map.with_index do |item, i|
       line_sgr = i == pointer ? enhanced_sgr : default_sgr
       str_width = dialog.width - (scrollbar_pos.nil? ? 0 : @block_elem_width)
-      str = padding_space_with_escape_sequences(Reline::Unicode.take_range(item, 0, str_width), str_width)
+      str, = Reline::Unicode.take_mbchar_range(item, 0, str_width, padding: true)
       colored_content = "#{line_sgr}#{str}"
       if scrollbar_pos
         if scrollbar_pos <= (i * 2) and (i * 2 + 1) < (scrollbar_pos + bar_height)

lib/reline/unicode.rb

@@ -179,32 +179,78 @@ def self.split_by_width(str, max_width, encoding = str.encoding, offset: 0)
 
   # Take a chunk of a String cut by width with escape sequences.
   def self.take_range(str, start_col, max_width)
+    take_mbchar_range(str, start_col, max_width).first
+  end
+
+  def self.take_mbchar_range(str, start_col, width, cover_begin: false, cover_end: false, padding: false)
     chunk = String.new(encoding: str.encoding)
+
+    end_col = start_col + width
     total_width = 0
     rest = str.encode(Encoding::UTF_8)
     in_zero_width = false
+    chunk_start_col = nil
+    chunk_end_col = nil
+    has_csi = false
     rest.scan(WIDTH_SCANNER) do |non_printing_start, non_printing_end, csi, osc, gc|
       case
       when non_printing_start
         in_zero_width = true
+        chunk << NON_PRINTING_START
       when non_printing_end
         in_zero_width = false
+        chunk << NON_PRINTING_END
       when csi
+        has_csi = true
         chunk << csi
       when osc
         chunk << osc
       when gc
         if in_zero_width
           chunk << gc
+          next
+        end
+
+        mbchar_width = get_mbchar_width(gc)
+        prev_width = total_width
+        total_width += mbchar_width
+
+        if (cover_begin || padding ? total_width <= start_col : prev_width < start_col)
+          # Current character haven't reached start_col yet
+          next
+        elsif padding && !cover_begin && prev_width < start_col && start_col < total_width
+          # Add preceding padding. This padding might have background color.
+          chunk << ' '
+          chunk_start_col ||= start_col
+          chunk_end_col = total_width
+          next
+        elsif (cover_end ? prev_width < end_col : total_width <= end_col)
+          # Current character is in the range
+          chunk << gc
+          chunk_start_col ||= prev_width
+          chunk_end_col = total_width
+          break if total_width >= end_col
         else
-          mbchar_width = get_mbchar_width(gc)
-          total_width += mbchar_width
-          break if (start_col + max_width) < total_width
-          chunk << gc if start_col < total_width
+          # Current character exceeds end_col
+          if padding && end_col < total_width
+            # Add succeeding padding. This padding might have background color.
+            chunk << ' '
+            chunk_start_col ||= prev_width
+            chunk_end_col = end_col
+          end
+          break
         end
       end
     end
-    chunk
+    chunk_start_col ||= start_col
+    chunk_end_col ||= start_col
+    if padding && chunk_end_col < end_col
+      # Append padding. This padding should not include background color.
+      chunk << "\e[0m" if has_csi
+      chunk << ' ' * (end_col - chunk_end_col)
+      chunk_end_col = end_col
+    end
+    [chunk, chunk_start_col, chunk_end_col - chunk_start_col]
   end
 
   def self.get_next_mbchar_size(line, byte_pointer)

test/reline/test_line_editor.rb

@@ -112,6 +112,36 @@ def test_dialog_move
       end
     end
 
+    def test_multibyte
+      base = [0, 12, '一二三一二三']
+      left = [0, 3, 'LLL']
+      right = [9, 3, 'RRR']
+      front = [3, 6, 'FFFFFF']
+      # 一 FFFFFF 三
+      # 一二三一二三
+      assert_output '[COL_2]二三一二' do
+        @line_editor.render_line_differential([base, front], [base, nil])
+      end
+
+      # LLLFFFFFF 三
+      # LLL 三一二三
+      assert_output '[COL_3] 三一二' do
+        @line_editor.render_line_differential([base, left, front], [base, left, nil])
+      end
+
+      # 一 FFFFFFRRR
+      # 一二三一 RRR
+      assert_output '[COL_2]二三一 ' do
+        @line_editor.render_line_differential([base, right, front], [base, right, nil])
+      end
+
+      # LLLFFFFFFRRR
+      # LLL 三一 RRR
+      assert_output '[COL_3] 三一 ' do
+        @line_editor.render_line_differential([base, left, right, front], [base, left, right, nil])
+      end
+    end
+
     def test_complicated
       state_a = [nil, [19, 7, 'bbbbbbb'], [15, 8, 'cccccccc'], [10, 5, 'ddddd'], [18, 4, 'eeee'], [1, 3, 'fff'], [17, 2, 'gg'], [7, 1, 'h']]
       state_b = [[5, 9, 'aaaaaaaaa'], nil, [15, 8, 'cccccccc'], nil, [18, 4, 'EEEE'], [25, 4, 'ffff'], [17, 2, 'gg'], [2, 2, 'hh']]

test/reline/test_unicode.rb

@@ -46,8 +46,8 @@ def test_split_by_width_csi_reset_sgr_optimization
   def test_take_range
     assert_equal 'cdef', Reline::Unicode.take_range('abcdefghi', 2, 4)
     assert_equal 'あde', Reline::Unicode.take_range('abあdef', 2, 4)
-    assert_equal 'zerocdef', Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4)
-    assert_equal 'bzerocde', Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4)
+    assert_equal "\1zero\2cdef", Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4)
+    assert_equal "b\1zero\2cde", Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4)
     assert_equal "\e[31mcd\e[42mef", Reline::Unicode.take_range("\e[31mabcd\e[42mefg", 2, 4)
     assert_equal "\e]0;1\acd", Reline::Unicode.take_range("ab\e]0;1\acd", 2, 3)
     assert_equal 'いう', Reline::Unicode.take_range('あいうえお', 2, 4)
@@ -67,4 +67,26 @@ def test_calculate_width
     assert_equal 10, Reline::Unicode.calculate_width('あいうえお')
     assert_equal 10, Reline::Unicode.calculate_width('あいうえお', true)
   end
+
+  def test_take_mbchar_range
+    assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4)
+    assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, padding: true)
+    assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_begin: true)
+    assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_end: true)
+    assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4)
+    assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, padding: true)
+    assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_begin: true)
+    assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_end: true)
+    assert_equal ['う', 4, 2], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4)
+    assert_equal [' う ', 3, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, padding: true)
+    assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true)
+    assert_equal ['うえ', 4, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true)
+    assert_equal ['いう ', 2, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true, padding: true)
+    assert_equal [' うえ', 3, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true, padding: true)
+    assert_equal [' うえお   ', 3, 10], Reline::Unicode.take_mbchar_range('あいうえお', 3, 10, padding: true)
+    assert_equal [" \e[41mうえお\e[0m   ", 3, 10], Reline::Unicode.take_mbchar_range("あい\e[41mうえお", 3, 10, padding: true)
+    assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true)
+    assert_equal ["\e[31mc\1ABC\2d\e[0mef", 2, 4], Reline::Unicode.take_mbchar_range("\e[31mabc\1ABC\2d\e[0mefghi", 2, 4)
+    assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true)
+  end
 end