Disable completions due to issues with non-ascii chars (#1917)

<!-- ELLIPSIS_HIDDEN -->


> [!IMPORTANT]
> Disables completion feature due to non-ASCII character issues and
updates position handling for multibyte characters in
`position_utils.rs`.
> 
>   - **Behavior**:
> - Disables completion feature in `completion.rs` due to issues with
non-ASCII characters, specifically Chinese/multibyte characters.
>     - Returns `None` for completion requests in `Completion::run()`.
>   - **Position Handling**:
> - Updates `is_first_inside_block()`, `get_word_at_position()`, and
`get_symbol_before_position()` in `position_utils.rs` to convert
character positions to byte offsets for accurate slicing.
> - Ensures character positions are clamped to line length to prevent
out-of-bounds errors.
>   - **Misc**:
> - Comments in `completion.rs` suggest conditions for re-enabling
completions, including testing on Windows and modifying position
handling for multibyte characters.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup>
for 4467798. You can
[customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this
summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

Author: aaronvg

engine/language_server/src/baml_project/position_utils.rs

@@ -5,7 +5,7 @@ use std::cmp;
 const MAX_SAFE_CHAR: u32 = std::u32::MAX; // You can adjust this to mimic your TS MAX_SAFE_VALUE_i32
 
 /// Returns the full range of the document, from the beginning (line 0, character 0)
-/// to the “end” (last line with a very large character position).
+/// to the "end" (last line with a very large character position).
 pub fn full_document_range(contents: &str) -> Range {
     // Compute the number of lines. If the text is empty, assume one line.
     let line_count = if contents.is_empty() {
@@ -42,15 +42,23 @@ pub fn get_current_line<'a>(contents: &'a str, line: u32) -> &'a str {
 /// The logic is as follows:
 /// - If the trimmed current line is empty, return true.
 /// - Otherwise, take the substring _up to_ the given position and check if the last
-///   “word” (using a `\w+` search) ends exactly at the position.
+///   "word" (using a `\w+` search) ends exactly at the position.
 pub fn is_first_inside_block(position: &Position, current_line: &str) -> bool {
     if current_line.trim().is_empty() {
         return true;
     }
 
-    // Ensure we don’t slice past the length of the current line.
-    let pos = cmp::min(position.character as usize, current_line.len());
-    let string_til_position = &current_line[..pos];
+    // Ensure we don't slice past the length of the current line, using character counts.
+    let char_count_in_line = current_line.chars().count();
+    let clamped_char_pos = cmp::min(position.character as usize, char_count_in_line);
+
+    // Convert character position to byte offset for slicing.
+    let byte_offset_at_clamped_char_pos = current_line
+        .char_indices()
+        .nth(clamped_char_pos)
+        .map_or(current_line.len(), |(idx, _)| idx);
+
+    let string_til_position = &current_line[..byte_offset_at_clamped_char_pos];
 
     // Find the first occurrence of a word.
     let re = Regex::new(r"\w+").unwrap();
@@ -71,21 +79,37 @@ pub fn is_first_inside_block(position: &Position, current_line: &str) -> bool {
 /// If no non-word boundary is found after the position, an empty string is returned.
 pub fn get_word_at_position(contents: &str, position: &Position) -> String {
     let current_line = get_current_line(contents, position.line);
-    let line_len = current_line.len();
+    if current_line.is_empty() {
+        return "".to_string();
+    }
 
-    // Clamp position.character to the current line length.
-    let pos = cmp::min(position.character as usize, line_len);
+    let char_count_in_line = current_line.chars().count();
+    // `position.character` is a 0-indexed character offset. Clamp it.
+    let clamped_char_idx = cmp::min(position.character as usize, char_count_in_line);
+
+    // Part 1: Search backward to find the start of the word.
+    // Slice for backward search extends up to character `clamped_char_idx + 1` (exclusive).
+    let char_len_for_backward_search = cmp::min(clamped_char_idx + 1, char_count_in_line);
+    let byte_len_for_backward_search_slice = current_line
+        .char_indices()
+        .nth(char_len_for_backward_search)
+        .map_or(current_line.len(), |(idx, _)| idx);
+    let text_for_backward_search = &current_line[..byte_len_for_backward_search_slice];
 
     // Search backward from position.character + 1 using a regex for the last non-whitespace sequence.
     let re_begin = Regex::new(r"\S+$").unwrap();
-    let slice_end = cmp::min(pos + 1, line_len);
-    let substring_before = &current_line[..slice_end];
-    let beginning = if let Some(mat) = re_begin.find(substring_before) {
-        mat.start()
-    } else {
-        return "".to_string();
+    let word_start_byte_idx = match re_begin.find(text_for_backward_search) {
+        Some(mat) => mat.start(),
+        None => return "".to_string(),
     };
 
+    // Convert `clamped_char_idx` (character index) to a byte index for the LSP `position.character`.
+    // This `pos` will be used as the starting point for the forward search.
+    let pos = current_line
+        .char_indices()
+        .nth(clamped_char_idx)
+        .map_or(current_line.len(), |(idx, _)| idx);
+
     // Search forward from position.character for the first non-word character.
     let re_end = Regex::new(r"\W").unwrap();
     let substring_after = &current_line[pos..];
@@ -97,8 +121,8 @@ pub fn get_word_at_position(contents: &str, position: &Position) -> String {
 
     let word_end = pos + end;
 
-    if beginning <= word_end && word_end <= current_line.len() {
-        current_line[beginning..word_end].to_string()
+    if word_start_byte_idx <= word_end && word_end <= current_line.len() {
+        current_line[word_start_byte_idx..word_end].to_string()
     } else {
         "".to_string()
     }
@@ -107,19 +131,44 @@ pub fn get_word_at_position(contents: &str, position: &Position) -> String {
 /// Returns the symbol (a single character) immediately preceding the given position.
 /// If the position is at the start of the line, an empty string is returned.
 pub fn get_symbol_before_position(contents: &str, position: &Position) -> String {
-    if position.character == 0 {
-        return "".to_string();
-    }
     let current_line = get_current_line(contents, position.line);
-    let pos = cmp::min(position.character as usize, current_line.len());
-    if pos == 0 {
+    // position.character is a 0-indexed character offset.
+    let char_cursor_pos = position.character as usize;
+
+    if char_cursor_pos == 0 {
         return "".to_string();
     }
-    // This simple slicing works correctly if the text is ASCII.
-    current_line[pos - 1..pos].to_string()
+
+    // Clamp the character cursor position against the actual number of characters in the line.
+    // This ensures that if position.character is, for example, 5, but the line only has 3 chars,
+    // we don't panic. We want the character at index (clamped_char_cursor_pos - 1).
+    let num_chars_in_line = current_line.chars().count();
+
+    // If the effective cursor position is beyond the line's character length,
+    // or if it's at the very beginning (char_cursor_pos == 0, handled above),
+    // there's no valid preceding character to get by simple indexing from char_cursor_pos.
+    // We are interested in the character at `char_cursor_pos - 1`.
+    if char_cursor_pos > num_chars_in_line {
+        // If cursor is effectively beyond the line, the "preceding" character would be the last one.
+        // So we try to get char at num_chars_in_line - 1.
+        if num_chars_in_line == 0 {
+            return "".to_string(); // Empty line
+        }
+        return current_line
+            .chars()
+            .nth(num_chars_in_line - 1)
+            .map_or("".to_string(), |ch| ch.to_string());
+    }
+
+    // At this point, 0 < char_cursor_pos <= num_chars_in_line.
+    // We want the character at index (char_cursor_pos - 1).
+    current_line
+        .chars()
+        .nth(char_cursor_pos - 1)
+        .map_or("".to_string(), |ch| ch.to_string())
 }
 
-/// Computes the Position (line and character) corresponding to a given index in the document’s text.
+/// Computes the Position (line and character) corresponding to a given index in the document's text.
 /// This mimics the TS implementation by iterating over each character up to the given index.
 pub fn get_position_from_index(document: &TextDocumentItem, index: usize) -> Position {
     let mut line: u32 = 0;

engine/language_server/src/server/api/requests/completion.rs

@@ -20,54 +20,59 @@ impl SyncRequestHandler for Completion {
         _requester: &mut Requester,
         params: CompletionParams,
     ) -> Result<Option<lsp_types::CompletionResponse>> {
-        let url = params.text_document_position.text_document.uri;
-        let path = url
-            .to_file_path()
-            .internal_error_msg("Could not convert URL to path")?;
+        // TODO: Enable this only if you
+        // 1. test on windows, with chinese characters
+        // 2. Modify position_utils.rs to use byte offsets to account for chinese/multibyte characters
+        // 3. Don't crash if you index into a string with a byte offset that is out of bounds
+        // let url = params.text_document_position.text_document.uri;
+        // let path = url
+        //     .to_file_path()
+        //     .internal_error_msg("Could not convert URL to path")?;
 
-        // Use the unified method to get or create the project
-        let project = session
-            .get_or_create_project(&path)
-            .expect("Failed to get or create project");
+        // // Use the unified method to get or create the project
+        // let project = session
+        //     .get_or_create_project(&path)
+        //     .expect("Failed to get or create project");
 
-        let guard = project.lock().unwrap();
-        let document_key =
-            DocumentKey::from_url(&PathBuf::from(guard.root_path()), &url).internal_error()?;
-        let doc = guard
-            .baml_project
-            .files
-            .get(&document_key)
-            .ok_or(anyhow::anyhow!(
-                "File {} was not present in the project",
-                document_key
-            ))
-            .internal_error()?;
-        let word = get_word_at_position(&doc.contents, &params.text_document_position.position);
-        let cleaned_word = trim_line(&word);
-        // let cleaned_word = word;
-        let completions = match cleaned_word.as_str() {
-            "_." => Some(vec![
-                r#"role("system")"#,
-                r#"role("assistant")"#,
-                r#"role("user")"#,
-            ]),
-            "ctx." => Some(vec![r#"output_format"#, r#"client"#]),
-            "ctx.client." => Some(vec![r#"name"#, r#"provider"#]),
-            _ => None,
-        };
-        Ok(completions.map(|completions| {
-            let completion_list = CompletionList {
-                is_incomplete: false,
-                items: completions
-                    .into_iter()
-                    .map(|completion| CompletionItem {
-                        label: completion.to_string(),
-                        ..CompletionItem::default()
-                    })
-                    .collect(),
-                ..CompletionList::default()
-            };
-            CompletionResponse::List(completion_list)
-        }))
+        // let guard = project.lock().unwrap();
+        // let document_key =
+        //     DocumentKey::from_url(&PathBuf::from(guard.root_path()), &url).internal_error()?;
+        // let doc = guard
+        //     .baml_project
+        //     .files
+        //     .get(&document_key)
+        //     .ok_or(anyhow::anyhow!(
+        //         "File {} was not present in the project",
+        //         document_key
+        //     ))
+        //     .internal_error()?;
+        // let word = get_word_at_position(&doc.contents, &params.text_document_position.position);
+        // let cleaned_word = trim_line(&word);
+        // // let cleaned_word = word;
+        // let completions = match cleaned_word.as_str() {
+        //     "_." => Some(vec![
+        //         r#"role("system")"#,
+        //         r#"role("assistant")"#,
+        //         r#"role("user")"#,
+        //     ]),
+        //     "ctx." => Some(vec![r#"output_format"#, r#"client"#]),
+        //     "ctx.client." => Some(vec![r#"name"#, r#"provider"#]),
+        //     _ => None,
+        // };
+        // Ok(completions.map(|completions| {
+        //     let completion_list = CompletionList {
+        //         is_incomplete: false,
+        //         items: completions
+        //             .into_iter()
+        //             .map(|completion| CompletionItem {
+        //                 label: completion.to_string(),
+        //                 ..CompletionItem::default()
+        //             })
+        //             .collect(),
+        //         ..CompletionList::default()
+        //     };
+        //     CompletionResponse::List(completion_list)
+        // }))
+        Ok(None)
     }
 }