Update tm4e upstream, fix issues with symbol matching, and improve completion item list sorting

editor/src/main/java/io/github/rosemoe/sora/lang/completion/CompletionPublisher.java

@@ -123,7 +123,7 @@ public void setComparator(@Nullable Comparator<CompletionItem> comparator) {
             return;
         }
         this.comparator = comparator;
-        if (items.size() != 0 && comparator != null) {
+        if (!items.isEmpty() && comparator != null) {
             handler.post(() -> {
                 if (invalid) {
                     return;
@@ -217,7 +217,7 @@ public void updateList(boolean forced) {
 
             if (locked) {
                 try {
-                    if (candidates.size() == 0) {
+                    if (candidates.isEmpty()) {
                         callback.run();
                         return;
                     }

editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt

@@ -30,21 +30,21 @@ import io.github.rosemoe.sora.text.CharPosition
 import io.github.rosemoe.sora.text.ContentReference
 import io.github.rosemoe.sora.util.CharCode
 
-fun fuzzyComparator(a: SortedCompletionItem, b: SortedCompletionItem): Int {
-    return if (a.score.score > b.score.score) {
-        -1;
-    } else if (a.score.score < b.score.score) {
-        1
-    } else {
-        snippetUpComparator(a.completionItem, b.completionItem)
-    }
-}
-
 private fun CharSequence?.asString(): String {
     return if (this == null) " " else if (this is String) this else this.toString()
 }
 
 fun defaultComparator(a: CompletionItem, b: CompletionItem): Int {
+    // check score
+    val p1Score = (a.extra as? SortedCompletionItem)?.score?.score ?: 0
+    val p2Score = (b.extra as? SortedCompletionItem)?.score?.score ?: 0
+
+    // if score biggest, it better similar to input text
+    if (p1Score < p2Score) {
+        return 1;
+    } else if (p1Score > p2Score) {
+        return -1;
+    }
 
     var p1 = a.sortText.asString()
     var p2 = b.sortText.asString()
@@ -67,25 +67,19 @@ fun defaultComparator(a: CompletionItem, b: CompletionItem): Int {
         return 1;
     }
 
-
     // check with 'kind'
+    // if kind biggest, it better important
+    val kind = (b.kind?.value ?: 0) - (a.kind?.value ?: 0)
 
-    val kind = (a.kind?.value ?: 0) - (b.kind?.value ?: 0)
-
-    return if (kind == 0) {
-        if (a.extra is SortedCompletionItem && b.extra is SortedCompletionItem) {
-            return fuzzyComparator(a.extra as SortedCompletionItem, b.extra as SortedCompletionItem)
-        } else kind
-    } else kind
-
+    return kind
 }
 
 fun snippetUpComparator(a: CompletionItem, b: CompletionItem): Int {
     if (a.kind != b.kind) {
         if (a.kind == CompletionItemKind.Snippet) {
-            return -1;
-        } else if (b.kind == CompletionItemKind.Snippet) {
             return 1;
+        } else if (b.kind == CompletionItemKind.Snippet) {
+            return -1;
         }
     }
     return defaultComparator(a, b);
@@ -100,7 +94,7 @@ fun getCompletionItemComparator(
 
     source.validateAccess()
 
-    val sourceLine = source.reference.getLine(cursorPosition.line);
+    val sourceLine = source.reference.getLine(cursorPosition.line)
 
     var word = ""
     var wordLow = ""
@@ -153,8 +147,14 @@ fun getCompletionItemComparator(
         // the fallback-sort using the initial sort order.
         // use a score of `-100` because that is out of the
         // bound of values `fuzzyScore` will return
-
-        if (wordLen > 0) {
+        if (wordLen == 0) {
+            // when there is nothing to score against, don't
+            // event try to do. Use a const rank and rely on
+            // the fallback-sort using the initial sort order.
+            // use a score of `-100` because that is out of the
+            // bound of values `fuzzyScore` will return
+            item.score = FuzzyScore.default
+        } else {
             // skip word characters that are whitespace until
             // we have hit the replace range (overwriteBefore)
             var wordPos = 0;
@@ -170,7 +170,42 @@ fun getCompletionItemComparator(
             if (wordPos >= wordLen) {
                 // the wordPos at which scoring starts is the whole word
                 // and therefore the same rules as not having a word apply
-                item.score = FuzzyScore.default
+                item.score = FuzzyScore.default;
+            } else if (originItem.sortText?.isNotEmpty() == true) {
+                // when there is a `filterText` it must match the `word`.
+                // if it matches we check with the label to compute highlights
+                // and if that doesn't yield a result we have no highlights,
+                // despite having the match
+                // by default match `word` against the `label`
+                val match = scoreFn.calculateScore(
+                    word,
+                    wordLow,
+                    wordPos,
+                    originItem.sortText.asString(),
+                    originItem.sortText.asString().lowercase(),
+                    0,
+                    FuzzyScoreOptions.default
+                ) ?: continue; // NO match
+
+                // compareIgnoreCase(item.completion.filterText, item.textLabel) === 0
+                if (originItem.sortText === originItem.label) {
+                    // filterText and label are actually the same -> use good highlights
+                    item.score = match;
+                } else {
+                    // re-run the scorer on the label in the hope of a result BUT use the rank
+                    // of the filterText-match
+                    val labelMatch = scoreFn.calculateScore(
+                        word,
+                        wordLow,
+                        wordPos,
+                        originItem.label.asString(),
+                        originItem.label.asString().lowercase(),
+                        0,
+                        FuzzyScoreOptions.default
+                    ) ?: continue; // NO match
+                    item.score = labelMatch
+                    labelMatch.matches[0] = match.matches[0]
+                }
 
             } else {
                 // by default match `word` against the `label`
@@ -185,12 +220,11 @@ fun getCompletionItemComparator(
                 ) ?: continue; // NO match
                 item.score = match;
             }
-        }
-
-        originItem.extra = item
-    }
 
+            originItem.extra = item
 
+        }
+    }
 
     return Comparator { o1, o2 ->
         snippetUpComparator(o1, o2)
@@ -200,4 +234,4 @@ fun getCompletionItemComparator(
 data class SortedCompletionItem(
     val completionItem: CompletionItem,
     var score: FuzzyScore
-)
+)

editor/src/main/java/io/github/rosemoe/sora/widget/CodeEditor.java

@@ -524,7 +524,7 @@ protected void initialize(AttributeSet attrs, int defStyleAttr, int defStyleRes)
             TypedArray a = null;
             try {
                 a = getContext().obtainStyledAttributes(new int[]{android.R.attr.listPreferredItemHeight});
-                verticalScrollFactor = a.getFloat(0, 32);
+                verticalScrollFactor = a.getDimension(0, 32);
             } catch (Exception e) {
                 Log.e(LOG_TAG, "Failed to get scroll factor, using default.", e);
                 verticalScrollFactor = 32;
@@ -1684,9 +1684,9 @@ public void indentSelection() {
     /**
      * Indents the lines. Does nothing if the <code>onlyIfSelected</code> is <code>true</code> and
      * no text is selected.
-     * 
+     *
      * @param onlyIfSelected Set to <code>true</code> if lines must be indented only if the text is
-     *                      selected.
+     *                       selected.
      */
     public void indentLines(boolean onlyIfSelected) {
 
@@ -1698,7 +1698,7 @@ public void indentLines(boolean onlyIfSelected) {
         final var tabString = createTabString();
         final var text = getText();
         final var tabWidth = getTabWidth();
-        
+
         text.beginBatchEdit();
         for (int i = cursor.getLeftLine(); i <= cursor.getRightLine(); i++) {
             final var line = text.getLine(i);
@@ -4134,9 +4134,9 @@ public void hideSoftInput() {
      * Check whether the soft keyboard is enabled for this editor. Unlike {@link #isSoftKeyboardEnabled()},
      * this method also checks whether a hardware keyboard is connected.
      *
+     * @return Whether the editor should show soft keyboard.
      * @see #isSoftKeyboardEnabled()
      * @see #isDisableSoftKbdIfHardKbdAvailable()
-     * @return Whether the editor should show soft keyboard.
      */
     protected boolean checkSoftInputEnabled() {
         if (isDisableSoftKbdIfHardKbdAvailable()

editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java

@@ -286,6 +286,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI
         } else if (composingText.isComposing()) {
             deleteComposingText();
         }
+
         // replace text
         SymbolPairMatch.SymbolPair pair = null;
         if (editor.getProps().symbolPairAutoCompletion && text.length() > 0) {
@@ -303,6 +304,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI
                     inputText, firstCharFromText
             );
         }
+
         var editorText = editor.getText();
         var editorCursor = editor.getCursor();
 
@@ -332,7 +334,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI
                 editor.setSelectionRegion(editorCursor.getLeftLine(), editorCursor.getLeftColumn(),
                         editorCursor.getRightLine(), editorCursor.getRightColumn() - pair.close.length());
             } else if (editorCursor.isSelected() && editor.getEditorLanguage().getQuickQuoteHandler() != null) {
-                editor.commitText(text,applyAutoIndent);
+                editor.commitText(text, applyAutoIndent);
             } else {
                 editorText.beginBatchEdit();
 

editor/src/main/java/io/github/rosemoe/sora/widget/component/DefaultCompletionLayout.java

@@ -25,12 +25,14 @@
 
 import android.animation.LayoutTransition;
 import android.content.Context;
+import android.graphics.Outline;
 import android.graphics.drawable.GradientDrawable;
 import android.os.SystemClock;
 import android.util.TypedValue;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewGroup;
+import android.view.ViewOutlineProvider;
 import android.widget.LinearLayout;
 import android.widget.ListView;
 import android.widget.ProgressBar;
@@ -115,6 +117,8 @@ public View inflate(@NonNull Context context) {
 
         rootLayout.setBackground(gd);
 
+        setRootViewOutlineProvider(rootView);
+
         listView.setDividerHeight(0);
         setLoading(true);
 
@@ -127,6 +131,8 @@ public View inflate(@NonNull Context context) {
             }
         });
 
+
+
         return rootLayout;
     }
 
@@ -137,6 +143,8 @@ public void onApplyColorScheme(@NonNull EditorColorScheme colorScheme) {
         gd.setStroke(1, colorScheme.getColor(EditorColorScheme.COMPLETION_WND_CORNER));
         gd.setColor(colorScheme.getColor(EditorColorScheme.COMPLETION_WND_BACKGROUND));
         rootView.setBackground(gd);
+
+        setRootViewOutlineProvider(rootView);
     }
 
     @Override
@@ -170,6 +178,16 @@ private void performScrollList(int offset) {
         ev.recycle();
     }
 
+    private void setRootViewOutlineProvider(View rootView) {
+        rootView.setOutlineProvider(new ViewOutlineProvider() {
+            @Override
+            public void getOutline(View view, Outline outline) {
+                outline.setRoundRect(0, 0, view.getWidth(), view.getHeight(), TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, 8, view.getContext().getResources().getDisplayMetrics()));
+            }
+        });
+        rootView.setClipToOutline(true);
+    }
+
     @Override
     public void ensureListPositionVisible(int position, int increment) {
         listView.post(() -> {

gradle/libs.versions.toml

@@ -28,7 +28,6 @@ jcodings = { module = "org.jruby.jcodings:jcodings", version = "1.0.58" }
 joni = { module = "org.jruby.joni:joni", version = "2.2.1" }
 snakeyaml-engine = { module = "org.snakeyaml:snakeyaml-engine", version = "2.2" }
 jdt-annotation = { module = "org.eclipse.jdt:org.eclipse.jdt.annotation", version = "2.2.800" }
-guava = { module = "com.google.guava:guava", version = "33.0.0-android" }
 
 tests-google-truth = { module = "com.google.truth:truth", version = "1.3.0" }
 tests-robolectric = { module = "org.robolectric:robolectric", version = "4.11.1" }

language-textmate/build.gradle.kts

@@ -65,7 +65,6 @@ dependencies {
 
     implementation(libs.snakeyaml.engine)
     implementation(libs.jdt.annotation)
-    implementation(libs.guava)
 
     testImplementation(libs.junit)
     androidTestImplementation(libs.androidx.test.junit)

language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateAnalyzer.java

@@ -29,6 +29,16 @@
 
 import androidx.annotation.NonNull;
 
+import org.eclipse.tm4e.core.grammar.IGrammar;
+import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes;
+import org.eclipse.tm4e.core.internal.grammar.tokenattrs.StandardTokenType;
+import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp;
+import org.eclipse.tm4e.core.internal.oniguruma.OnigResult;
+import org.eclipse.tm4e.core.internal.oniguruma.OnigString;
+import org.eclipse.tm4e.core.internal.theme.FontStyle;
+import org.eclipse.tm4e.core.internal.theme.Theme;
+import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration;
+
 import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
@@ -47,18 +57,6 @@
 import io.github.rosemoe.sora.langs.textmate.registry.model.ThemeModel;
 import io.github.rosemoe.sora.langs.textmate.utils.StringUtils;
 import io.github.rosemoe.sora.text.Content;
-
-import org.eclipse.tm4e.core.grammar.IGrammar;
-
-import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes;
-import org.eclipse.tm4e.core.internal.grammar.tokenattrs.StandardTokenType;
-import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp;
-import org.eclipse.tm4e.core.internal.oniguruma.OnigResult;
-import org.eclipse.tm4e.core.internal.oniguruma.OnigString;
-import org.eclipse.tm4e.core.internal.theme.FontStyle;
-import org.eclipse.tm4e.core.internal.theme.Theme;
-import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration;
-
 import io.github.rosemoe.sora.text.ContentLine;
 import io.github.rosemoe.sora.text.ContentReference;
 import io.github.rosemoe.sora.util.ArrayList;