Fix the handling of multibyte characters

lib/ruby_lsp/document.rb

@@ -146,16 +146,20 @@ def locate(node, char_position, node_types: [])
 
         # Skip if the current node doesn't cover the desired position
         loc = candidate.location
-        next unless (loc.start_offset...loc.end_offset).cover?(char_position)
+        next unless (loc.start_code_units_offset(encoding)...loc.end_code_units_offset(encoding)).cover?(char_position)
 
         # If the node's start character is already past the position, then we should've found the closest node
         # already
-        break if char_position < loc.start_offset
+        break if char_position < loc.start_code_units_offset(encoding)
 
         # If the candidate starts after the end of the previous nesting level, then we've exited that nesting level and
         # need to pop the stack
         previous_level = nesting_nodes.last
-        nesting_nodes.pop if previous_level && loc.start_offset > previous_level.location.end_offset
+        if previous_level
+          start_offset = loc.start_code_units_offset(encoding)
+          end_offset = previous_level.location.end_code_units_offset(encoding)
+          nesting_nodes.pop if start_offset > end_offset
+        end
 
         # Keep track of the nesting where we found the target. This is used to determine the fully qualified name of the
         # target when it is a constant
@@ -182,7 +186,9 @@ def locate(node, char_position, node_types: [])
 
         # If the current node is narrower than or equal to the previous closest node, then it is more precise
         closest_loc = closest.location
-        if loc.end_offset - loc.start_offset <= closest_loc.end_offset - closest_loc.start_offset
+        target_length = loc.end_code_units_offset(encoding) - loc.start_code_units_offset(encoding)
+        closest_length = closest_loc.end_code_units_offset(encoding) - closest_loc.start_code_units_offset(encoding)
+        if target_length <= closest_length
           parent = closest
           closest = candidate
         end

lib/ruby_lsp/requests/definition.rb

@@ -75,9 +75,10 @@ def initialize(document, global_state, position, dispatcher, typechecker_enabled
             target,
             parent,
             position,
+            global_state.encoding,
           )
         elsif target.is_a?(Prism::CallNode) && target.name != :require && target.name != :require_relative &&
-            !covers_position?(target.message_loc, position)
+            !covers_position?(target.message_loc, position, global_state.encoding)
           # If the target is a method call, we need to ensure that the requested position is exactly on top of the
           # method identifier. Otherwise, we risk showing definitions for unrelated things
           target = nil

lib/ruby_lsp/requests/hover.rb

@@ -52,9 +52,10 @@ def initialize(document, global_state, position, dispatcher, typechecker_enabled
             T.must(target),
             T.must(parent),
             position,
+            global_state.encoding,
           )
         elsif target.is_a?(Prism::CallNode) && target.name != :require && target.name != :require_relative &&
-            !covers_position?(target.message_loc, position)
+            !covers_position?(target.message_loc, position, global_state.encoding)
 
           target = nil
         end

lib/ruby_lsp/requests/request.rb

@@ -18,19 +18,19 @@ def perform; end
       private
 
       # Checks if a location covers a position
-      sig { params(location: Prism::Location, position: T.untyped).returns(T::Boolean) }
-      def cover?(location, position)
+      sig { params(location: Prism::Location, position: T.untyped, encoding: Encoding).returns(T::Boolean) }
+      def cover?(location, position, encoding)
         start_covered =
           location.start_line - 1 < position[:line] ||
           (
             location.start_line - 1 == position[:line] &&
-              location.start_column <= position[:character]
+              location.start_code_units_column(encoding) <= position[:character]
           )
         end_covered =
           location.end_line - 1 > position[:line] ||
           (
             location.end_line - 1 == position[:line] &&
-              location.end_column >= position[:character]
+              location.end_code_units_column(encoding) >= position[:character]
           )
         start_covered && end_covered
       end
@@ -50,15 +50,16 @@ def cover?(location, position)
           target: Prism::Node,
           parent: Prism::Node,
           position: T::Hash[Symbol, Integer],
+          encoding: Encoding,
         ).returns(Prism::Node)
       end
-      def determine_target(target, parent, position)
+      def determine_target(target, parent, position, encoding)
         return target unless parent.is_a?(Prism::ConstantPathNode)
 
         target = T.let(parent, Prism::Node)
         parent = T.let(T.cast(target, Prism::ConstantPathNode).parent, T.nilable(Prism::Node))
 
-        while parent && cover?(parent.location, position)
+        while parent && cover?(parent.location, position, encoding)
           target = parent
           parent = target.is_a?(Prism::ConstantPathNode) ? target.parent : nil
         end
@@ -67,17 +68,23 @@ def determine_target(target, parent, position)
       end
 
       # Checks if a given location covers the position requested
-      sig { params(location: T.nilable(Prism::Location), position: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
-      def covers_position?(location, position)
+      sig do
+        params(
+          location: T.nilable(Prism::Location),
+          position: T::Hash[Symbol, T.untyped],
+          encoding: Encoding,
+        ).returns(T::Boolean)
+      end
+      def covers_position?(location, position, encoding)
         return false unless location
 
         start_line = location.start_line - 1
         end_line = location.end_line - 1
         line = position[:line]
         character = position[:character]
 
-        (start_line < line || (start_line == line && location.start_column <= character)) &&
-          (end_line > line || (end_line == line && location.end_column >= character))
+        (start_line < line || (start_line == line && location.start_code_units_column(encoding) <= character)) &&
+          (end_line > line || (end_line == line && location.end_code_units_column(encoding) >= character))
       end
     end
   end

test/expectations/definition/multibyte_characters_reference.exp.json

@@ -0,0 +1,23 @@
+{
+    "result": [
+        {
+            "uri": "file:///fixtures/multibyte_characters.rb",
+            "range": {
+                "start": {
+                    "line": 2,
+                    "character": 2
+                },
+                "end": {
+                    "line": 8,
+                    "character": 5
+                }
+            }
+        }
+    ],
+    "params": [
+        {
+            "line": 1,
+            "character": 7
+        }
+    ]
+}

test/fixtures/multibyte_characters_reference.rb

@@ -0,0 +1,2 @@
+# テスト
+A動物::C貓咪.叫

test/requests/definition_expectations_test.rb

@@ -41,6 +41,15 @@ def run_expectations(source)
           ),
         ),
       )
+      index.index_single(
+        RubyIndexer::IndexablePath.new(
+          "#{Dir.pwd}/lib",
+          File.expand_path(
+            "../../test/fixtures/multibyte_characters.rb",
+            __dir__,
+          ),
+        ),
+      )
 
       server.process_message(
         id: 1,

test/ruby_document_test.rb

@@ -558,6 +558,23 @@ def baz
     assert_equal(["Foo", "Bar"], node_context.nesting)
   end
 
+  def test_locate_when_multibyte_characters
+    document = RubyLsp::RubyDocument.new(source: <<~RUBY, version: 1, uri: URI("file:///foo/bar.rb"))
+      module A動物
+        class B猫
+          def 鳴く
+            puts "にゃー"
+          end
+        end
+      end
+    RUBY
+
+    node_context = document.locate_node({ line: 1, character: 8 })
+    found = node_context.node
+    assert_equal("B猫", T.cast(found, Prism::ConstantReadNode).location.slice)
+    assert_equal(["A動物", "B猫"], node_context.nesting)
+  end
+
   def test_reparsing_without_new_edits_does_nothing
     text = "def foo; end"