From c51eb1622822b6ecd129c2ec379ef6af9cb6ad24 Mon Sep 17 00:00:00 2001 From: dingyi Date: Mon, 29 Jan 2024 20:32:36 +0800 Subject: [PATCH 1/9] feat(lang-textmate): merge upstream of tm4e update & fix #537 Update taken from https://github.com/eclipse/tm4e/commit/1d2beb8576ea552a9da18710cc72a9a537497f16 --- gradle/libs.versions.toml | 1 - language-textmate/build.gradle.kts | 1 - .../sora/langs/textmate/TextMateAnalyzer.java | 22 +- .../langs/textmate/TextMateColorScheme.java | 6 - .../sora/langs/textmate/TextMateLanguage.java | 2 +- .../textmate/TextMateNewlineHandler.java | 39 +- .../textmate/TextMateSymbolPairMatch.java | 7 +- .../textmate/registry/GrammarRegistry.java | 2 +- .../tm4e/core/internal/grammar/Grammar.java | 644 ++++++------ .../core/internal/grammar/LineTokenizer.java | 984 +++++++++--------- .../core/internal/grammar/LineTokens.java | 370 ++++--- .../grammar/raw/RawGrammarReader.java | 2 +- .../core/internal/matcher/MatcherBuilder.java | 3 +- .../internal/oniguruma/OnigCaptureIndex.java | 4 +- .../core/internal/oniguruma/OnigRegExp.java | 65 +- .../core/internal/oniguruma/OnigResult.java | 19 +- .../internal/parser/PropertySettable.java | 48 +- .../core/internal/parser/TMParserPList.java | 261 +++-- .../internal/parser/TMParserPropertyPath.java | 4 +- .../core/internal/parser/TMParserYAML.java | 30 +- .../core/internal/registry/SyncRegistry.java | 18 +- .../tm4e/core/internal/rule/RegExpSource.java | 320 +++--- .../tm4e/core/internal/rule/RuleFactory.java | 19 +- .../tm4e/core/internal/theme/ColorMap.java | 40 +- .../tm4e/core/internal/theme/FontStyle.java | 38 +- .../tm4e/core/internal/theme/Theme.java | 469 ++++----- .../internal/utils/AbstractListeners.java | 52 +- .../core/internal/utils/MoreCollections.java | 16 +- .../core/internal/utils/ObjectCloner.java | 36 +- .../tm4e/core/internal/utils/RegexSource.java | 12 +- .../tm4e/core/internal/utils/ScopeNames.java | 49 + .../tm4e/core/internal/utils/StringUtils.java | 306 +++--- .../tm4e/core/registry/IGrammarSource.java | 180 ++-- .../tm4e/core/registry/IThemeSource.java | 3 +- .../eclipse/tm4e/core/registry/Registry.java | 30 +- .../internal/model/AutoClosingPair.java | 30 + .../model/AutoClosingPairConditional.java | 39 + .../internal/model/CharacterPair.java | 39 + .../internal/model/CommentRule.java | 49 + .../internal/model/CompleteEnterAction.java | 43 + .../internal/model/EnterAction.java | 100 ++ .../internal/model/FoldingRules.java | 47 + .../internal/model/IndentForEnter.java | 45 + .../internal/model/IndentationRules.java | 58 ++ .../internal/model/LanguageConfiguration.java | 457 ++++++++ .../internal/model/OnEnterRule.java | 78 ++ .../internal/model/RegExPattern.java | 136 +++ .../supports/CharacterPairSupport.java | 94 ++ .../internal/supports/CommentSupport.java | 83 ++ .../internal/supports/IndentRulesSupport.java | 79 ++ .../internal/supports/OnEnterSupport.java | 139 +++ .../internal/utils/RegExpUtils.java | 52 + .../internal/utils/TextUtils.java | 216 ++++ .../model/AutoClosingPair.java | 36 - .../model/AutoClosingPairConditional.java | 41 - .../model/CharacterPair.java | 62 -- .../model/CommentRule.java | 53 - .../model/CompleteEnterAction.java | 44 - .../model/EnterAction.java | 104 -- .../model/FoldingRules.java | 51 - .../model/IndentationRules.java | 64 -- .../model/LanguageConfiguration.java | 427 -------- .../model/OnEnterRule.java | 69 -- .../supports/CharacterPairSupport.java | 104 -- .../supports/CommentSupport.java | 96 -- .../supports/IndentRulesSupport.java | 102 -- .../supports/OnEnterSupport.java | 163 --- .../utils/RegExpUtils.java | 58 -- .../utils/TabSpacesInfo.java | 43 - .../utils/TextUtils.java | 164 --- 70 files changed, 3848 insertions(+), 3719 deletions(-) create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ScopeNames.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPair.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CommentRule.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CompleteEnterAction.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/EnterAction.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/FoldingRules.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentForEnter.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentationRules.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/OnEnterRule.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/RegExPattern.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CharacterPairSupport.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CommentSupport.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/IndentRulesSupport.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/OnEnterSupport.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/RegExpUtils.java create mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/TextUtils.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPair.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPairConditional.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CharacterPair.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CommentRule.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CompleteEnterAction.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/EnterAction.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/FoldingRules.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/IndentationRules.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/LanguageConfiguration.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/OnEnterRule.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CharacterPairSupport.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CommentSupport.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/IndentRulesSupport.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/OnEnterSupport.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/RegExpUtils.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TabSpacesInfo.java delete mode 100644 language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TextUtils.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 54e20aaed..516447477 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,7 +28,6 @@ jcodings = { module = "org.jruby.jcodings:jcodings", version = "1.0.58" } joni = { module = "org.jruby.joni:joni", version = "2.2.1" } snakeyaml-engine = { module = "org.snakeyaml:snakeyaml-engine", version = "2.2" } jdt-annotation = { module = "org.eclipse.jdt:org.eclipse.jdt.annotation", version = "2.2.800" } -guava = { module = "com.google.guava:guava", version = "33.0.0-android" } tests-google-truth = { module = "com.google.truth:truth", version = "1.3.0" } tests-robolectric = { module = "org.robolectric:robolectric", version = "4.11.1" } diff --git a/language-textmate/build.gradle.kts b/language-textmate/build.gradle.kts index 65119c871..4f74a3401 100644 --- a/language-textmate/build.gradle.kts +++ b/language-textmate/build.gradle.kts @@ -65,7 +65,6 @@ dependencies { implementation(libs.snakeyaml.engine) implementation(libs.jdt.annotation) - implementation(libs.guava) testImplementation(libs.junit) androidTestImplementation(libs.androidx.test.junit) diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateAnalyzer.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateAnalyzer.java index facc42015..1778c6a70 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateAnalyzer.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateAnalyzer.java @@ -29,6 +29,16 @@ import androidx.annotation.NonNull; +import org.eclipse.tm4e.core.grammar.IGrammar; +import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes; +import org.eclipse.tm4e.core.internal.grammar.tokenattrs.StandardTokenType; +import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp; +import org.eclipse.tm4e.core.internal.oniguruma.OnigResult; +import org.eclipse.tm4e.core.internal.oniguruma.OnigString; +import org.eclipse.tm4e.core.internal.theme.FontStyle; +import org.eclipse.tm4e.core.internal.theme.Theme; +import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration; + import java.time.Duration; import java.util.Collections; import java.util.List; @@ -47,18 +57,6 @@ import io.github.rosemoe.sora.langs.textmate.registry.model.ThemeModel; import io.github.rosemoe.sora.langs.textmate.utils.StringUtils; import io.github.rosemoe.sora.text.Content; - -import org.eclipse.tm4e.core.grammar.IGrammar; - -import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes; -import org.eclipse.tm4e.core.internal.grammar.tokenattrs.StandardTokenType; -import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp; -import org.eclipse.tm4e.core.internal.oniguruma.OnigResult; -import org.eclipse.tm4e.core.internal.oniguruma.OnigString; -import org.eclipse.tm4e.core.internal.theme.FontStyle; -import org.eclipse.tm4e.core.internal.theme.Theme; -import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration; - import io.github.rosemoe.sora.text.ContentLine; import io.github.rosemoe.sora.text.ContentReference; import io.github.rosemoe.sora.util.ArrayList; diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateColorScheme.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateColorScheme.java index d8ccab3af..f3145aa45 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateColorScheme.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateColorScheme.java @@ -109,12 +109,10 @@ public void applyDefault() { RawTheme rawSubTheme; if (settings == null) { - Log.e("theme", "load vscode theme"); rawSubTheme = ((RawTheme) ((RawTheme) rawTheme).get("colors")); applyVSCTheme(rawSubTheme); } else { - Log.e("theme", "load default theme"); rawSubTheme = (RawTheme) ((List) settings).get(0); rawSubTheme = (RawTheme) rawSubTheme.getSetting(); @@ -127,8 +125,6 @@ public void applyDefault() { private void applyVSCTheme(RawTheme RawTheme) { - Log.e("vsc theme", - Arrays.toString(Collections.singletonList(RawTheme).toArray())); setColor(LINE_DIVIDER, Color.TRANSPARENT); String caret = (String) RawTheme.get("editorCursor.foreground"); @@ -153,8 +149,6 @@ private void applyVSCTheme(RawTheme RawTheme) { } String background = (String) RawTheme.get("editor.background"); - Log.e("vsc forge", RawTheme.toString()); - Log.e("vsc ??", background + "??"); if (background != null) { setColor(WHOLE_BACKGROUND, Color.parseColor(background)); setColor(LINE_NUMBER_BACKGROUND, Color.parseColor(background)); diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java index b28b8e810..1b65ac7ba 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java @@ -32,7 +32,7 @@ import org.eclipse.tm4e.core.grammar.IGrammar; import org.eclipse.tm4e.core.registry.IGrammarSource; import org.eclipse.tm4e.core.registry.IThemeSource; -import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration; +import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration; import java.io.Reader; import java.util.Objects; diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateNewlineHandler.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateNewlineHandler.java index ea86b2cfb..ce6763511 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateNewlineHandler.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateNewlineHandler.java @@ -29,22 +29,19 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import org.eclipse.tm4e.languageconfiguration.model.CompleteEnterAction; -import org.eclipse.tm4e.languageconfiguration.model.EnterAction; -import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration; -import org.eclipse.tm4e.languageconfiguration.supports.IndentRulesSupport; -import org.eclipse.tm4e.languageconfiguration.supports.OnEnterSupport; -import org.eclipse.tm4e.languageconfiguration.utils.TabSpacesInfo; + +import org.eclipse.tm4e.languageconfiguration.internal.model.CompleteEnterAction; +import org.eclipse.tm4e.languageconfiguration.internal.model.EnterAction; +import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration; +import org.eclipse.tm4e.languageconfiguration.internal.supports.IndentRulesSupport; +import org.eclipse.tm4e.languageconfiguration.internal.supports.OnEnterSupport; +import org.eclipse.tm4e.languageconfiguration.internal.utils.TextUtils; import java.util.Arrays; -import java.util.regex.Pattern; import io.github.rosemoe.sora.lang.smartEnter.NewlineHandleResult; import io.github.rosemoe.sora.lang.smartEnter.NewlineHandler; import io.github.rosemoe.sora.lang.styling.Styles; - -import org.eclipse.tm4e.languageconfiguration.utils.TextUtils; - import io.github.rosemoe.sora.text.CharPosition; import io.github.rosemoe.sora.text.Content; @@ -149,7 +146,7 @@ public NewlineHandleResult handleNewline(@NonNull Content text, @NonNull CharPos return new NewlineHandleResult(typeText, caretOffset); } case Outdent: - final var indentation = TextUtils.getIndentationFromWhitespace(enterAction.indentation, getTabSpaces()); + final var indentation = TextUtils.getIndentationFromWhitespace(enterAction.indentation, language.getTabSize(), language.useTab()); final var outdentedText = outdentString(normalizeIndentation(indentation + enterAction.appendText)); var caretOffset = outdentedText.length() + 1; @@ -179,7 +176,6 @@ protected Pair getIndentForEnter(Content text, CharPosition posi var afterEnterAction = getInheritIndentForLine(new WrapperContentImp(text, position.line, beforeEnterText), true, position.line + 1); - var tabSpaces = TextUtils.getTabSpaces(language); if (afterEnterAction == null) { return new Pair<>(beforeEnterIndent, beforeEnterIndent); @@ -188,8 +184,8 @@ protected Pair getIndentForEnter(Content text, CharPosition posi var afterEnterIndent = afterEnterAction.indentation; var indent = ""; - if (tabSpaces.isInsertSpaces()) { - indent = " ".repeat(tabSpaces.getTabSize()); + if (language.useTab()) { + indent = " ".repeat(language.getTabSize()); } else { indent = "\t"; } @@ -435,7 +431,7 @@ public CompleteEnterAction getEnterAction(final Content content, final CharPosit indentation = indentation.substring(0, indentation.length() - removeText); } - return new CompleteEnterAction(enterResult, indentation); + return new CompleteEnterAction(enterResult.indentAction, enterResult.appendText, enterResult.removeText, indentation); } @@ -444,9 +440,9 @@ private String outdentString(final String str) { if (str.startsWith("\t")) { // $NON-NLS-1$ return str.substring(1); } - final TabSpacesInfo tabSpaces = getTabSpaces(); - if (tabSpaces.isInsertSpaces()) { - final char[] chars = new char[tabSpaces.getTabSize()]; + + if (language.useTab()) { + final char[] chars = new char[language.getTabSize()]; Arrays.fill(chars, ' '); final String spaces = new String(chars); if (str.startsWith(spaces)) { @@ -458,12 +454,7 @@ private String outdentString(final String str) { private String normalizeIndentation(final String str) { - final TabSpacesInfo tabSpaces = getTabSpaces(); - return TextUtils.normalizeIndentation(str, tabSpaces.getTabSize(), tabSpaces.isInsertSpaces()); - } - - private TabSpacesInfo getTabSpaces() { - return TextUtils.getTabSpaces(language); + return TextUtils.normalizeIndentation(str, language.getTabSize(), language.useTab()); } diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java index ac274bcf7..347beac59 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java @@ -23,16 +23,11 @@ */ package io.github.rosemoe.sora.langs.textmate; -import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes; import org.eclipse.tm4e.core.internal.grammar.tokenattrs.StandardTokenType; -import org.eclipse.tm4e.languageconfiguration.model.AutoClosingPairConditional; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import io.github.rosemoe.sora.lang.styling.Span; import io.github.rosemoe.sora.text.Content; @@ -40,6 +35,8 @@ import io.github.rosemoe.sora.widget.CodeEditor; import io.github.rosemoe.sora.widget.SymbolPairMatch; +import org.eclipse.tm4e.languageconfiguration.internal.model.AutoClosingPairConditional; + public class TextMateSymbolPairMatch extends SymbolPairMatch { private static final String surroundingPairFlag = "surroundingPair"; diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java index 5859af4ba..e1e99063b 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java @@ -30,7 +30,7 @@ import org.eclipse.tm4e.core.registry.IGrammarSource; import org.eclipse.tm4e.core.registry.IThemeSource; import org.eclipse.tm4e.core.registry.Registry; -import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration; +import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration; import java.io.InputStreamReader; import java.io.Reader; diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java index 0de736a2a..aad7e3b72 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java @@ -3,13 +3,13 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Initial code from https://github.com/microsoft/vscode-textmate/ * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. * Initial license: MIT - *

+ * * Contributors: * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java @@ -17,6 +17,16 @@ */ package org.eclipse.tm4e.core.internal.grammar; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.grammar.IGrammar; import org.eclipse.tm4e.core.grammar.IStateStack; @@ -38,158 +48,148 @@ import org.eclipse.tm4e.core.internal.utils.ObjectCloner; import org.eclipse.tm4e.core.internal.utils.StringUtils; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Function; - import io.github.rosemoe.sora.util.Logger; /** * TextMate grammar implementation. * * @see - * github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/grammar/grammar.ts#L98"> + * github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts */ public final class Grammar implements IGrammar, IRuleFactoryHelper { private static final Logger LOGGER = Logger.instance(Grammar.class.getName()); - private final String rootScopeName; - - @Nullable - private RuleId _rootId; - private int _lastRuleId = 0; - private final Map _ruleId2desc = new HashMap<>(); - private final Map includedGrammars = new HashMap<>(); - private final IGrammarRepository _grammarRepository; - private final IRawGrammar _grammar; - final IThemeProvider themeProvider; - - @Nullable - private List _injections; - private final BasicScopeAttributesProvider _basicScopeAttributesProvider; - private final List _tokenTypeMatchers = new ArrayList<>(); - - @Nullable - private final BalancedBracketSelectors balancedBracketSelectors; - - public Grammar( - final String rootScopeName, - final IRawGrammar grammar, - final int initialLanguage, - @Nullable final Map embeddedLanguages, - @Nullable final Map tokenTypes, - @Nullable final BalancedBracketSelectors balancedBracketSelectors, - final IGrammarRepository grammarRepository, - final IThemeProvider themeProvider) { - - this.rootScopeName = rootScopeName; - this._basicScopeAttributesProvider = new BasicScopeAttributesProvider(initialLanguage, embeddedLanguages); - this._grammarRepository = grammarRepository; - this._grammar = initGrammar(grammar, null); - this.balancedBracketSelectors = balancedBracketSelectors; - this.themeProvider = themeProvider; - - if (tokenTypes != null) { - for (final var entry : tokenTypes.entrySet()) { - final var selector = entry.getKey(); - final var type = entry.getValue(); - for (final var matcher : Matcher.createMatchers(selector)) { - _tokenTypeMatchers.add(new TokenTypeMatcher(matcher.matcher, type)); - } - } - } - } - - BasicScopeAttributes getMetadataForScope(final String scope) { - return this._basicScopeAttributesProvider.getBasicScopeAttributes(scope); - } - - private void collectInjections(final List result, final String selector, final IRawRule rule, - final IRuleFactoryHelper ruleFactoryHelper, final IRawGrammar grammar) { - final var matchers = Matcher.createMatchers(selector); - final var ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, this._grammar.getRepository()); - for (final var matcher : matchers) { - result.add(new Injection( - selector, - matcher.matcher, - ruleId, - grammar, - matcher.priority)); - } - } - - private List _collectInjections() { - final var grammarRepository = new IGrammarRepository() { - @Override - public @Nullable IRawGrammar lookup(final String scopeName) { - if (Objects.equals(scopeName, Grammar.this.rootScopeName)) { - return Grammar.this._grammar; - } - return getExternalGrammar(scopeName, null); - } - - @Override - public @Nullable Collection injections(final String targetScope) { - return Grammar.this._grammarRepository.injections(targetScope); - } - }; - - final var result = new ArrayList(); - - final var scopeName = this.rootScopeName; - - final var grammar = grammarRepository.lookup(scopeName); - if (grammar != null) { - // add injections from the current grammar - final var rawInjections = grammar.getInjections(); - if (rawInjections != null) { - for (final var e : rawInjections.entrySet()) { - collectInjections( - result, - e.getKey(), - e.getValue(), - this, - grammar); - } - } - - // add injection grammars contributed for the current scope - final var injectionScopeNames = this._grammarRepository.injections(scopeName); - if (injectionScopeNames != null) { - injectionScopeNames.forEach(injectionScopeName -> { - final var injectionGrammar = Grammar.this.getExternalGrammar(injectionScopeName, null); - if (injectionGrammar != null) { - final var selector = injectionGrammar.getInjectionSelector(); - if (selector != null) { - collectInjections( - result, - selector, - injectionGrammar.toRawRule(), - this, - injectionGrammar); - } - } - }); - } - } - - Collections.sort(result, (i1, i2) -> i1.priority - i2.priority); // sort by priority - - return result; - } - - List getInjections() { - var injections = this._injections; - if (injections == null) { - injections = this._injections = this._collectInjections(); + private final String rootScopeName; + + @Nullable + private RuleId _rootId; + private int _lastRuleId = 0; + private final Map _ruleId2desc = new HashMap<>(); + private final Map includedGrammars = new HashMap<>(); + private final IGrammarRepository _grammarRepository; + private final IRawGrammar _grammar; + final IThemeProvider themeProvider; + + @Nullable + private List _injections; + private final BasicScopeAttributesProvider _basicScopeAttributesProvider; + private final List _tokenTypeMatchers = new ArrayList<>(); + + @Nullable + private final BalancedBracketSelectors balancedBracketSelectors; + + public Grammar( + final String rootScopeName, + final IRawGrammar grammar, + final int initialLanguage, + @Nullable final Map embeddedLanguages, + @Nullable final Map tokenTypes, + @Nullable final BalancedBracketSelectors balancedBracketSelectors, + final IGrammarRepository grammarRepository, + final IThemeProvider themeProvider) { + + this.rootScopeName = rootScopeName; + this._basicScopeAttributesProvider = new BasicScopeAttributesProvider(initialLanguage, embeddedLanguages); + this._grammarRepository = grammarRepository; + this._grammar = initGrammar(grammar, null); + this.balancedBracketSelectors = balancedBracketSelectors; + this.themeProvider = themeProvider; + + if (tokenTypes != null) { + for (final var entry : tokenTypes.entrySet()) { + final var selector = entry.getKey(); + final var type = entry.getValue(); + for (final var matcher : Matcher.createMatchers(selector)) { + _tokenTypeMatchers.add(new TokenTypeMatcher(matcher.matcher, type)); + } + } + } + } + + BasicScopeAttributes getMetadataForScope(final String scope) { + return this._basicScopeAttributesProvider.getBasicScopeAttributes(scope); + } + + private void collectInjections(final List result, final String selector, final IRawRule rule, + final IRuleFactoryHelper ruleFactoryHelper, final IRawGrammar grammar) { + final var matchers = Matcher.createMatchers(selector); + final var ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, this._grammar.getRepository()); + for (final var matcher : matchers) { + result.add(new Injection( + selector, + matcher.matcher, + ruleId, + grammar, + matcher.priority)); + } + } + + private List _collectInjections() { + final var grammarRepository = new IGrammarRepository() { + @Override + public @Nullable IRawGrammar lookup(final String scopeName) { + if (Objects.equals(scopeName, Grammar.this.rootScopeName)) { + return Grammar.this._grammar; + } + return getExternalGrammar(scopeName, null); + } + + @Override + public @Nullable Collection injections(final String targetScope) { + return Grammar.this._grammarRepository.injections(targetScope); + } + }; + + final var result = new ArrayList(); + + final var scopeName = this.rootScopeName; + + final var grammar = grammarRepository.lookup(scopeName); + if (grammar != null) { + // add injections from the current grammar + final var rawInjections = grammar.getInjections(); + if (rawInjections != null) { + for (final var e : rawInjections.entrySet()) { + collectInjections( + result, + e.getKey(), + e.getValue(), + this, + grammar); + } + } + + // add injection grammars contributed for the current scope + final var injectionScopeNames = this._grammarRepository.injections(scopeName); + if (injectionScopeNames != null) { + injectionScopeNames.forEach(injectionScopeName -> { + final var injectionGrammar = Grammar.this.getExternalGrammar(injectionScopeName, null); + if (injectionGrammar != null) { + final var selector = injectionGrammar.getInjectionSelector(); + if (selector != null) { + collectInjections( + result, + selector, + injectionGrammar.toRawRule(), + this, + injectionGrammar); + } + } + }); + } + } + + Collections.sort(result, (i1, i2) -> i1.priority - i2.priority); // sort by priority + + return result; + } + + List getInjections() { + var injections = this._injections; + if (injections == null) { + injections = this._injections = this._collectInjections(); // remove ?? if (/*LOGGER && */!injections.isEmpty()) { @@ -202,182 +202,182 @@ List getInjections() { return injections; } - @Override - public T registerRule(final Function factory) { - final var id = RuleId.of(++this._lastRuleId); - final @Nullable T result = factory.apply(id); - this._ruleId2desc.put(id, result); - return result; - } - - @Override - public Rule getRule(final RuleId ruleId) { - final var rule = this._ruleId2desc.get(ruleId); - if (rule == null) { - throw new IndexOutOfBoundsException( - "No rule with index " + ruleId.id + " found. Possible values: 0.." + this._ruleId2desc.size()); - } - return rule; - } - - @Override - @Nullable - public IRawGrammar getExternalGrammar(final String scopeName, @Nullable final IRawRepository repository) { - if (this.includedGrammars.containsKey(scopeName)) { - return this.includedGrammars.get(scopeName); - } - - final IRawGrammar rawIncludedGrammar = this._grammarRepository.lookup(scopeName); - if (rawIncludedGrammar != null) { - this.includedGrammars.put(scopeName, initGrammar( - rawIncludedGrammar, - repository != null ? repository.getBase() : null)); - return this.includedGrammars.get(scopeName); - } - return null; - } - - private IRawGrammar initGrammar(IRawGrammar grammar, @Nullable final IRawRule base) { - grammar = ObjectCloner.deepClone(grammar); - - final var repo = grammar.getRepository(); - repo.setSelf(new RawRule() - .setName(grammar.getScopeName()) - .setPatterns(grammar.getPatterns())); - repo.setBase(base != null ? base : repo.getSelf()); - return grammar; - } - - @Override - public ITokenizeLineResult tokenizeLine(final String lineText) { - return tokenizeLine(lineText, null, null); - } - - @Override - public ITokenizeLineResult tokenizeLine(final String lineText, - @Nullable final IStateStack prevState, - @Nullable final Duration timeLimit) { - return _tokenize(lineText, (StateStack) prevState, false, timeLimit); - } - - @Override - public ITokenizeLineResult tokenizeLine2(final String lineText) { - return tokenizeLine2(lineText, null, null); - } - - @Override - public ITokenizeLineResult tokenizeLine2(final String lineText, @Nullable final IStateStack prevState, - @Nullable final Duration timeLimit) { - return _tokenize(lineText, (StateStack) prevState, true, timeLimit); - } - - @SuppressWarnings("unchecked") - private T _tokenize( - String lineText, - @Nullable StateStack prevState, - final boolean emitBinaryTokens, - @Nullable final Duration timeLimit) { - var rootId = this._rootId; - if (rootId == null) { - rootId = this._rootId = RuleFactory.getCompiledRuleId( - this._grammar.getRepository().getSelf(), - this, - this._grammar.getRepository()); - // This ensures ids are deterministic, and thus equal in renderer and webworker. - this.getInjections(); - } - - final boolean isFirstLine; - if (prevState == null || prevState == StateStack.NULL) { - isFirstLine = true; - final var rawDefaultMetadata = this._basicScopeAttributesProvider.getDefaultAttributes(); - final var defaultStyle = this.themeProvider.getDefaults(); - final int defaultMetadata = EncodedTokenAttributes.set( - 0, - rawDefaultMetadata.languageId, - rawDefaultMetadata.tokenType, - null, - defaultStyle.fontStyle, - defaultStyle.foregroundId, - defaultStyle.backgroundId); - - final var rootScopeName = this.getRule(rootId).getName(null, null); - - final AttributedScopeStack scopeList; - if (rootScopeName != null) { - scopeList = AttributedScopeStack.createRootAndLookUpScopeName( - rootScopeName, - defaultMetadata, - this); - } else { - scopeList = AttributedScopeStack.createRoot( - "unknown", - defaultMetadata); - } - - prevState = new StateStack( - null, - rootId, - -1, - -1, - false, - null, - scopeList, - scopeList); - } else { - isFirstLine = false; - prevState.reset(); - } - - if (lineText.isEmpty() || lineText.charAt(lineText.length() - 1) != '\n') { - // Only add \n if the passed lineText didn't have it. - lineText += '\n'; - } - final var onigLineText = OnigString.of(lineText); - final int lineLength = lineText.length(); - final var lineTokens = new LineTokens( - emitBinaryTokens, - lineText, - _tokenTypeMatchers, - balancedBracketSelectors); - final var r = LineTokenizer.tokenizeString( - this, - onigLineText, - isFirstLine, - 0, - prevState, - lineTokens, - true, - timeLimit == null ? Duration.ZERO : timeLimit); - - return (T) new TokenizeLineResult<>( - emitBinaryTokens - ? lineTokens.getBinaryResult(r.stack, lineLength) - : lineTokens.getResult(r.stack, lineLength), - r.stack, - r.stoppedEarly); - } - - @Override - @Nullable - public String getName() { - return _grammar.getName(); - } - - @Override - public String getScopeName() { - return rootScopeName; - } - - @Override - public Collection getFileTypes() { - return _grammar.getFileTypes(); - } - - @Override - public String toString() { - return StringUtils.toString(this, sb -> sb - .append("name=").append(getName()).append(", ") - .append("scopeName=").append(getScopeName())); - } + @Override + public T registerRule(final Function factory) { + final var id = RuleId.of(++this._lastRuleId); + final @Nullable T result = factory.apply(id); + this._ruleId2desc.put(id, result); + return result; + } + + @Override + public Rule getRule(final RuleId ruleId) { + final var rule = this._ruleId2desc.get(ruleId); + if (rule == null) { + throw new IndexOutOfBoundsException( + "No rule with index " + ruleId.id + " found. Possible values: 0.." + this._ruleId2desc.size()); + } + return rule; + } + + @Override + @Nullable + public IRawGrammar getExternalGrammar(final String scopeName, @Nullable final IRawRepository repository) { + if (this.includedGrammars.containsKey(scopeName)) { + return this.includedGrammars.get(scopeName); + } + + final IRawGrammar rawIncludedGrammar = this._grammarRepository.lookup(scopeName); + if (rawIncludedGrammar != null) { + this.includedGrammars.put(scopeName, initGrammar( + rawIncludedGrammar, + repository != null ? repository.getBase() : null)); + return this.includedGrammars.get(scopeName); + } + return null; + } + + private IRawGrammar initGrammar(IRawGrammar grammar, @Nullable final IRawRule base) { + grammar = ObjectCloner.deepClone(grammar); + + final var repo = grammar.getRepository(); + repo.setSelf(new RawRule() + .setName(grammar.getScopeName()) + .setPatterns(grammar.getPatterns())); + repo.setBase(base != null ? base : repo.getSelf()); + return grammar; + } + + @Override + public ITokenizeLineResult tokenizeLine(final String lineText) { + return tokenizeLine(lineText, null, null); + } + + @Override + public ITokenizeLineResult tokenizeLine(final String lineText, + @Nullable final IStateStack prevState, + @Nullable final Duration timeLimit) { + return _tokenize(lineText, (StateStack) prevState, false, timeLimit); + } + + @Override + public ITokenizeLineResult tokenizeLine2(final String lineText) { + return tokenizeLine2(lineText, null, null); + } + + @Override + public ITokenizeLineResult tokenizeLine2(final String lineText, @Nullable final IStateStack prevState, + @Nullable final Duration timeLimit) { + return _tokenize(lineText, (StateStack) prevState, true, timeLimit); + } + + @SuppressWarnings("unchecked") + private T _tokenize( + String lineText, + @Nullable StateStack prevState, + final boolean emitBinaryTokens, + @Nullable final Duration timeLimit) { + var rootId = this._rootId; + if (rootId == null) { + rootId = this._rootId = RuleFactory.getCompiledRuleId( + this._grammar.getRepository().getSelf(), + this, + this._grammar.getRepository()); + // This ensures ids are deterministic, and thus equal in renderer and webworker. + this.getInjections(); + } + + final boolean isFirstLine; + if (prevState == null || prevState == StateStack.NULL) { + isFirstLine = true; + final var rawDefaultMetadata = this._basicScopeAttributesProvider.getDefaultAttributes(); + final var defaultStyle = this.themeProvider.getDefaults(); + final int defaultMetadata = EncodedTokenAttributes.set( + 0, + rawDefaultMetadata.languageId, + rawDefaultMetadata.tokenType, + null, + defaultStyle.fontStyle, + defaultStyle.foregroundId, + defaultStyle.backgroundId); + + final var rootScopeName = this.getRule(rootId).getName(null, null); + + final AttributedScopeStack scopeList; + if (rootScopeName != null) { + scopeList = AttributedScopeStack.createRootAndLookUpScopeName( + rootScopeName, + defaultMetadata, + this); + } else { + scopeList = AttributedScopeStack.createRoot( + "unknown", + defaultMetadata); + } + + prevState = new StateStack( + null, + rootId, + -1, + -1, + false, + null, + scopeList, + scopeList); + } else { + isFirstLine = false; + prevState.reset(); + } + + if (lineText.isEmpty() || lineText.charAt(lineText.length() - 1) != '\n') { + // Only add \n if the passed lineText didn't have it. + lineText += '\n'; + } + final var onigLineText = OnigString.of(lineText); + final int lineLength = lineText.length(); + final var lineTokens = new LineTokens( + emitBinaryTokens, + lineText, + _tokenTypeMatchers, + balancedBracketSelectors); + final var r = LineTokenizer.tokenizeString( + this, + onigLineText, + isFirstLine, + 0, + prevState, + lineTokens, + true, + timeLimit == null ? Duration.ZERO : timeLimit); + + return (T) new TokenizeLineResult<>( + emitBinaryTokens + ? lineTokens.getBinaryResult(r.stack, lineLength) + : lineTokens.getResult(r.stack, lineLength), + r.stack, + r.stoppedEarly); + } + + @Override + @Nullable + public String getName() { + return _grammar.getName(); + } + + @Override + public String getScopeName() { + return rootScopeName; + } + + @Override + public Collection getFileTypes() { + return _grammar.getFileTypes(); + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("name=").append(getName()).append(", ") + .append("scopeName=").append(getScopeName())); + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java index afb3ce287..71844517c 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java @@ -3,21 +3,29 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Initial code from https://github.com/microsoft/vscode-textmate/ * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. * Initial license: MIT - *

+ * * Contributors: * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java */ package org.eclipse.tm4e.core.internal.grammar; + import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.NonNullByDefault; import org.eclipse.jdt.annotation.Nullable; @@ -32,12 +40,6 @@ import org.eclipse.tm4e.core.internal.rule.Rule; import org.eclipse.tm4e.core.internal.rule.RuleId; -import java.time.Duration; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - import io.github.rosemoe.sora.util.Logger; /** @@ -49,340 +51,340 @@ final class LineTokenizer { private static final Logger LOGGER = Logger.instance(LineTokenizer.class.getName()); - private record LocalStackElement(AttributedScopeStack scopes, int endPos) { - } - - private static class MatchResult { - final OnigCaptureIndex[] captureIndices; - final RuleId matchedRuleId; - - MatchResult(final RuleId matchedRuleId, final OnigCaptureIndex[] captureIndices) { - this.matchedRuleId = matchedRuleId; - this.captureIndices = captureIndices; - } - } - - private static final class MatchInjectionsResult extends MatchResult { - final boolean isPriorityMatch; - - MatchInjectionsResult(final RuleId matchedRuleId, final OnigCaptureIndex[] captureIndices, final boolean isPriorityMatch) { - super(matchedRuleId, captureIndices); - this.isPriorityMatch = isPriorityMatch; - } - } - - @NonNullByDefault({}) - private record WhileCheckResult( - @NonNull StateStack stack, - int linePos, - int anchorPosition, - boolean isFirstLine) { - } - - static final class TokenizeStringResult { - public final StateStack stack; - public final boolean stoppedEarly; - - TokenizeStringResult(final StateStack stack, final boolean stoppedEarly) { - this.stack = stack; - this.stoppedEarly = stoppedEarly; - } - } - - private final Grammar grammar; - private final OnigString lineText; - private boolean isFirstLine; - private int linePos; - private StateStack stack; - private final LineTokens lineTokens; - private int anchorPosition = -1; - private boolean stop; - - private LineTokenizer(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final int linePos, - final StateStack stack, final LineTokens lineTokens) { - this.grammar = grammar; - this.lineText = lineText; - this.isFirstLine = isFirstLine; - this.linePos = linePos; - this.stack = stack; - this.lineTokens = lineTokens; - } - - private TokenizeStringResult scan(final boolean checkWhileConditions, final long timeLimit) { - stop = false; - - if (checkWhileConditions) { - final var whileCheckResult = checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); - stack = whileCheckResult.stack; - linePos = whileCheckResult.linePos; - isFirstLine = whileCheckResult.isFirstLine; - anchorPosition = whileCheckResult.anchorPosition; - } - - final var startTime = System.currentTimeMillis(); - while (!stop) { - if (timeLimit > 0) { - final var elapsedTime = System.currentTimeMillis() - startTime; - if (elapsedTime > timeLimit) { - return new TokenizeStringResult(stack, true); - } - } - scanNext(); // potentially modifies linePos && anchorPosition - } - - return new TokenizeStringResult(stack, false); - } + private record LocalStackElement(AttributedScopeStack scopes, int endPos) { + } + + private static class MatchResult { + final OnigCaptureIndex[] captureIndices; + final RuleId matchedRuleId; + + MatchResult(final RuleId matchedRuleId, final OnigCaptureIndex[] captureIndices) { + this.matchedRuleId = matchedRuleId; + this.captureIndices = captureIndices; + } + } + + private static final class MatchInjectionsResult extends MatchResult { + final boolean isPriorityMatch; + + MatchInjectionsResult(final RuleId matchedRuleId, final OnigCaptureIndex[] captureIndices, final boolean isPriorityMatch) { + super(matchedRuleId, captureIndices); + this.isPriorityMatch = isPriorityMatch; + } + } + + @NonNullByDefault({}) + private record WhileCheckResult( + @NonNull StateStack stack, + int linePos, + int anchorPosition, + boolean isFirstLine) { + } + + static final class TokenizeStringResult { + public final StateStack stack; + public final boolean stoppedEarly; + + TokenizeStringResult(final StateStack stack, final boolean stoppedEarly) { + this.stack = stack; + this.stoppedEarly = stoppedEarly; + } + } + + private final Grammar grammar; + private final OnigString lineText; + private boolean isFirstLine; + private int linePos; + private StateStack stack; + private final LineTokens lineTokens; + private int anchorPosition = -1; + private boolean stop; + + private LineTokenizer(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final int linePos, + final StateStack stack, final LineTokens lineTokens) { + this.grammar = grammar; + this.lineText = lineText; + this.isFirstLine = isFirstLine; + this.linePos = linePos; + this.stack = stack; + this.lineTokens = lineTokens; + } + + private TokenizeStringResult scan(final boolean checkWhileConditions, final long timeLimit) { + stop = false; + + if (checkWhileConditions) { + final var whileCheckResult = checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); + stack = whileCheckResult.stack; + linePos = whileCheckResult.linePos; + isFirstLine = whileCheckResult.isFirstLine; + anchorPosition = whileCheckResult.anchorPosition; + } + + final var startTime = System.currentTimeMillis(); + while (!stop) { + if (timeLimit > 0) { + final var elapsedTime = System.currentTimeMillis() - startTime; + if (elapsedTime > timeLimit) { + return new TokenizeStringResult(stack, true); + } + } + scanNext(); // potentially modifies linePos && anchorPosition + } + + return new TokenizeStringResult(stack, false); + } private void scanNext() { // LOGGER.log(TRACE, () -> "@@scanNext: |" + lineText.content.replace("\n", "\\n").substring(linePos) + '|'); - final MatchResult r = matchRuleOrInjections(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); + final MatchResult r = matchRuleOrInjections(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); if (r == null) { - // LOGGER.log(TRACE, " no more matches."); + // LOGGER.log(TRACE, " no more matches."); // No match lineTokens.produce(stack, lineText.content.length()); stop = true; return; } - final OnigCaptureIndex[] captureIndices = r.captureIndices; - final RuleId matchedRuleId = r.matchedRuleId; - final boolean hasAdvanced = captureIndices.length > 0 && captureIndices[0].end > linePos; + final OnigCaptureIndex[] captureIndices = r.captureIndices; + final RuleId matchedRuleId = r.matchedRuleId; + final boolean hasAdvanced = captureIndices.length > 0 && captureIndices[0].end > linePos; - if (matchedRuleId.equals(RuleId.END_RULE)) { - // We matched the `end` for this rule => pop it - final BeginEndRule poppedRule = (BeginEndRule) stack.getRule(grammar); + if (matchedRuleId.equals(RuleId.END_RULE)) { + // We matched the `end` for this rule => pop it + final BeginEndRule poppedRule = (BeginEndRule) stack.getRule(grammar); - /* - * if (logger.isEnabled()) { logger.log(" popping " + poppedRule.debugName + - * " - " + poppedRule.debugEndRegExp); } - */ + /* + * if (logger.isEnabled()) { logger.log(" popping " + poppedRule.debugName + + * " - " + poppedRule.debugEndRegExp); } + */ - lineTokens.produce(stack, captureIndices[0].start); - stack = stack.withContentNameScopesList(stack.nameScopesList); - handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, poppedRule.endCaptures, captureIndices); - lineTokens.produce(stack, captureIndices[0].end); + lineTokens.produce(stack, captureIndices[0].start); + stack = stack.withContentNameScopesList(stack.nameScopesList); + handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, poppedRule.endCaptures, captureIndices); + lineTokens.produce(stack, captureIndices[0].end); - // pop - final var popped = stack; - stack = castNonNull(stack.pop()); - anchorPosition = popped.getAnchorPos(); + // pop + final var popped = stack; + stack = castNonNull(stack.pop()); + anchorPosition = popped.getAnchorPos(); if (!hasAdvanced && popped.getEnterPos() == linePos) { // Grammar pushed & popped a rule without advancing - // LOGGER.log(INFO, "[1] - Grammar is in an endless loop - Grammar pushed & popped a rule without advancing"); + // LOGGER.log(INFO, "[1] - Grammar is in an endless loop - Grammar pushed & popped a rule without advancing"); // See https://github.com/microsoft/vscode-textmate/issues/12 // Let's assume this was a mistake by the grammar author and the // intent was to continue in this state stack = popped; - lineTokens.produce(stack, lineText.content.length()); - stop = true; - return; - } - } else if (captureIndices.length > 0) { - // We matched a rule! - final Rule rule = grammar.getRule(matchedRuleId); - - lineTokens.produce(stack, captureIndices[0].start); - - final StateStack beforePush = stack; - // push it on the stack rule - final var scopeName = rule.getName(lineText.content, captureIndices); - final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar); - stack = stack.push( - matchedRuleId, - linePos, - anchorPosition, - captureIndices[0].end == lineText.content.length(), - null, - nameScopesList, - nameScopesList); - - if (rule instanceof final BeginEndRule pushedRule) { + lineTokens.produce(stack, lineText.content.length()); + stop = true; + return; + } + } else if (captureIndices.length > 0) { + // We matched a rule! + final Rule rule = grammar.getRule(matchedRuleId); + + lineTokens.produce(stack, captureIndices[0].start); + + final StateStack beforePush = stack; + // push it on the stack rule + final var scopeName = rule.getName(lineText.content, captureIndices); + final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar); + stack = stack.push( + matchedRuleId, + linePos, + anchorPosition, + captureIndices[0].end == lineText.content.length(), + null, + nameScopesList, + nameScopesList); + + if (rule instanceof final BeginEndRule pushedRule) { /*if(LOGGER.isLoggable(DEBUG)) { LOGGER.log(DEBUG, " pushing " + pushedRule.debugName + " - " + pushedRule.debugBeginRegExp); }*/ - handleCaptures( - grammar, - lineText, - isFirstLine, - stack, - lineTokens, - pushedRule.beginCaptures, - captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - anchorPosition = captureIndices[0].end; - - final var contentName = pushedRule.getContentName(lineText.content, captureIndices); - final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); - stack = stack.withContentNameScopesList(contentNameScopesList); - - if (pushedRule.endHasBackReferences) { - stack = stack.withEndRule( - pushedRule.getEndWithResolvedBackReferences( - lineText.content, - captureIndices)); - } - - if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { - // Grammar pushed the same rule without advancing - //LOGGER.log(INFO, "[2] - Grammar is in an endless loop - Grammar pushed the same rule without advancing"); - stack = castNonNull(stack.pop()); - lineTokens.produce(stack, lineText.content.length()); - stop = true; - return; - } - } else if (rule instanceof final BeginWhileRule pushedRule) { - // if (DebugFlags.InDebugMode) { - // console.log(" pushing " + pushedRule.debugName); - // } - - handleCaptures( - grammar, - lineText, - isFirstLine, - stack, - lineTokens, - pushedRule.beginCaptures, - captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - anchorPosition = captureIndices[0].end; - final var contentName = pushedRule.getContentName(lineText.content, captureIndices); - final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); - stack = stack.withContentNameScopesList(contentNameScopesList); - - if (pushedRule.whileHasBackReferences) { - stack = stack.withEndRule( - pushedRule.getWhileWithResolvedBackReferences( - lineText.content, - captureIndices)); - } - - if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { - // Grammar pushed the same rule without advancing - // LOGGER.log(INFO, "[3] - Grammar is in an endless loop - Grammar pushed the same rule without advancing"); - stack = castNonNull(stack.pop()); - lineTokens.produce(stack, lineText.content.length()); - stop = true; - return; - } - } else { - final MatchRule matchingRule = (MatchRule) rule; - // if (DebugFlags.InDebugMode) { - // console.log(' matched ' + matchingRule.debugName + ' - ' + - // matchingRule.debugMatchRegExp); - // } - - handleCaptures( - grammar, - lineText, - isFirstLine, - stack, - lineTokens, - matchingRule.captures, - captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - - // pop rule immediately since it is a MatchRule - stack = castNonNull(stack.pop()); - - if (!hasAdvanced) { - // Grammar is not advancing, nor is it pushing/popping - // LOGGER.log(INFO, "[4] - Grammar is in an endless loop - Grammar is not advancing, nor is it pushing/popping"); - stack = stack.safePop(); - lineTokens.produce(stack, lineText.content.length()); - stop = true; - return; - } - } - } - - if (captureIndices.length > 0 && captureIndices[0].end > linePos) { - // Advance stream - linePos = captureIndices[0].end; - isFirstLine = false; - } - } - - @Nullable - private MatchResult matchRule(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final int linePos, - final StateStack stack, final int anchorPosition) { - final var rule = stack.getRule(grammar); - final var ruleScanner = rule.compileAG(grammar, stack.endRule, isFirstLine, linePos == anchorPosition); - - final OnigScannerMatch r = ruleScanner.scanner.findNextMatch(lineText, linePos); - - if (r != null) { - return new MatchResult(ruleScanner.rules[r.index], r.getCaptureIndices()); - } - return null; - } - - @Nullable - private MatchResult matchRuleOrInjections(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, - final int linePos, final StateStack stack, final int anchorPosition) { - // Look for normal grammar rule - final MatchResult matchResult = matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); - - // Look for injected rules - final List injections = grammar.getInjections(); - if (injections.isEmpty()) { - // No injections whatsoever => early return - return matchResult; - } - - final var injectionResult = matchInjections(injections, grammar, lineText, isFirstLine, linePos, stack, anchorPosition); - if (injectionResult == null) { - // No injections matched => early return - return matchResult; - } - - if (matchResult == null) { - // Only injections matched => early return - return injectionResult; - } - - // Decide if `matchResult` or `injectionResult` should win - final int matchResultScore = matchResult.captureIndices[0].start; - final int injectionResultScore = injectionResult.captureIndices[0].start; - - if (injectionResultScore < matchResultScore || injectionResult.isPriorityMatch && injectionResultScore == matchResultScore) { - // injection won! - return injectionResult; - } - - return matchResult; - } - - @Nullable - private MatchInjectionsResult matchInjections(final List injections, final Grammar grammar, final OnigString lineText, - final boolean isFirstLine, final int linePos, final StateStack stack, final int anchorPosition) { - - // The lower the better - var bestMatchRating = Integer.MAX_VALUE; - OnigCaptureIndex[] bestMatchCaptureIndices = null; - var bestMatchRuleId = RuleId.END_RULE; - var bestMatchResultPriority = 0; - - final List scopes = stack.contentNameScopesList != null ? stack.contentNameScopesList.getScopeNames() - : Collections.emptyList(); - - for (int i = 0, len = injections.size(); i < len; i++) { - final var injection = injections.get(i); - if (!injection.matches(scopes)) { - // injection selector doesn't match stack - continue; - } - - final var rule = grammar.getRule(injection.ruleId); - final var ruleScanner = rule.compileAG(grammar, null, isFirstLine, linePos == anchorPosition); - final var matchResult = ruleScanner.scanner.findNextMatch(lineText, linePos); - if (matchResult == null) { - continue; - } + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + pushedRule.beginCaptures, + captureIndices); + lineTokens.produce(stack, captureIndices[0].end); + anchorPosition = captureIndices[0].end; + + final var contentName = pushedRule.getContentName(lineText.content, captureIndices); + final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); + stack = stack.withContentNameScopesList(contentNameScopesList); + + if (pushedRule.endHasBackReferences) { + stack = stack.withEndRule( + pushedRule.getEndWithResolvedBackReferences( + lineText.content, + captureIndices)); + } + + if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { + // Grammar pushed the same rule without advancing + // LOGGER.log(INFO, "[2] - Grammar is in an endless loop - Grammar pushed the same rule without advancing"); + stack = castNonNull(stack.pop()); + lineTokens.produce(stack, lineText.content.length()); + stop = true; + return; + } + } else if (rule instanceof final BeginWhileRule pushedRule) { + // if (DebugFlags.InDebugMode) { + // console.log(" pushing " + pushedRule.debugName); + // } + + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + pushedRule.beginCaptures, + captureIndices); + lineTokens.produce(stack, captureIndices[0].end); + anchorPosition = captureIndices[0].end; + final var contentName = pushedRule.getContentName(lineText.content, captureIndices); + final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); + stack = stack.withContentNameScopesList(contentNameScopesList); + + if (pushedRule.whileHasBackReferences) { + stack = stack.withEndRule( + pushedRule.getWhileWithResolvedBackReferences( + lineText.content, + captureIndices)); + } + + if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { + // Grammar pushed the same rule without advancing + // LOGGER.log(INFO, "[3] - Grammar is in an endless loop - Grammar pushed the same rule without advancing"); + stack = castNonNull(stack.pop()); + lineTokens.produce(stack, lineText.content.length()); + stop = true; + return; + } + } else { + final MatchRule matchingRule = (MatchRule) rule; + // if (DebugFlags.InDebugMode) { + // console.log(' matched ' + matchingRule.debugName + ' - ' + + // matchingRule.debugMatchRegExp); + // } + + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + matchingRule.captures, + captureIndices); + lineTokens.produce(stack, captureIndices[0].end); + + // pop rule immediately since it is a MatchRule + stack = castNonNull(stack.pop()); + + if (!hasAdvanced) { + // Grammar is not advancing, nor is it pushing/popping + // LOGGER.log(INFO, "[4] - Grammar is in an endless loop - Grammar is not advancing, nor is it pushing/popping"); + stack = stack.safePop(); + lineTokens.produce(stack, lineText.content.length()); + stop = true; + return; + } + } + } + + if (captureIndices.length > 0 && captureIndices[0].end > linePos) { + // Advance stream + linePos = captureIndices[0].end; + isFirstLine = false; + } + } + + @Nullable + private MatchResult matchRule(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final int linePos, + final StateStack stack, final int anchorPosition) { + final var rule = stack.getRule(grammar); + final var ruleScanner = rule.compileAG(grammar, stack.endRule, isFirstLine, linePos == anchorPosition); + + final OnigScannerMatch r = ruleScanner.scanner.findNextMatch(lineText, linePos); + + if (r != null) { + return new MatchResult(ruleScanner.rules[r.index], r.getCaptureIndices()); + } + return null; + } + + @Nullable + private MatchResult matchRuleOrInjections(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, + final int linePos, final StateStack stack, final int anchorPosition) { + // Look for normal grammar rule + final MatchResult matchResult = matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); + + // Look for injected rules + final List injections = grammar.getInjections(); + if (injections.isEmpty()) { + // No injections whatsoever => early return + return matchResult; + } + + final var injectionResult = matchInjections(injections, grammar, lineText, isFirstLine, linePos, stack, anchorPosition); + if (injectionResult == null) { + // No injections matched => early return + return matchResult; + } + + if (matchResult == null) { + // Only injections matched => early return + return injectionResult; + } + + // Decide if `matchResult` or `injectionResult` should win + final int matchResultScore = matchResult.captureIndices[0].start; + final int injectionResultScore = injectionResult.captureIndices[0].start; + + if (injectionResultScore < matchResultScore || injectionResult.isPriorityMatch && injectionResultScore == matchResultScore) { + // injection won! + return injectionResult; + } + + return matchResult; + } + + @Nullable + private MatchInjectionsResult matchInjections(final List injections, final Grammar grammar, final OnigString lineText, + final boolean isFirstLine, final int linePos, final StateStack stack, final int anchorPosition) { + + // The lower the better + var bestMatchRating = Integer.MAX_VALUE; + OnigCaptureIndex[] bestMatchCaptureIndices = null; + var bestMatchRuleId = RuleId.END_RULE; + var bestMatchResultPriority = 0; + + final List scopes = stack.contentNameScopesList != null ? stack.contentNameScopesList.getScopeNames() + : Collections.emptyList(); + + for (int i = 0, len = injections.size(); i < len; i++) { + final var injection = injections.get(i); + if (!injection.matches(scopes)) { + // injection selector doesn't match stack + continue; + } + + final var rule = grammar.getRule(injection.ruleId); + final var ruleScanner = rule.compileAG(grammar, null, isFirstLine, linePos == anchorPosition); + final var matchResult = ruleScanner.scanner.findNextMatch(lineText, linePos); + if (matchResult == null) { + continue; + } final int matchRating = matchResult.getCaptureIndices()[0].start; if (matchRating > bestMatchRating) { @@ -390,180 +392,180 @@ private MatchInjectionsResult matchInjections(final List injections, continue; } - bestMatchRating = matchRating; - bestMatchCaptureIndices = matchResult.getCaptureIndices(); - bestMatchRuleId = ruleScanner.rules[matchResult.index]; - bestMatchResultPriority = injection.priority; - - if (bestMatchRating == linePos) { - // No more need to look at the rest of the injections - break; - } - } - - if (bestMatchCaptureIndices != null) { - return new MatchInjectionsResult( - bestMatchRuleId, - bestMatchCaptureIndices, - bestMatchResultPriority == -1); - } - - return null; - } - - private void handleCaptures(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final StateStack stack, - final LineTokens lineTokens, final List<@Nullable CaptureRule> captures, final OnigCaptureIndex[] captureIndices) { - if (captures.isEmpty()) { - return; - } - - final var lineTextContent = lineText.content; - - final int len = Math.min(captures.size(), captureIndices.length); - final var localStack = new ArrayDeque(); - final int maxEnd = captureIndices[0].end; - - for (int i = 0; i < len; i++) { - final var captureRule = captures.get(i); - if (captureRule == null) { - // Not interested - continue; - } - - final var captureIndex = captureIndices[i]; - - if (captureIndex.getLength() == 0) { - // Nothing really captured - continue; - } - - if (captureIndex.start > maxEnd) { - // Capture going beyond consumed string - break; - } - - // pop captures while needed - while (!localStack.isEmpty() && localStack.getLast().endPos <= captureIndex.start) { - // pop! - final var lastElem = localStack.removeLast(); - lineTokens.produceFromScopes(lastElem.scopes, lastElem.endPos); - } - - if (!localStack.isEmpty()) { - lineTokens.produceFromScopes(localStack.getLast().scopes, captureIndex.start); - } else { - lineTokens.produce(stack, captureIndex.start); - } - - final var retokenizeCapturedWithRuleId = captureRule.retokenizeCapturedWithRuleId; - if (retokenizeCapturedWithRuleId.notEquals(RuleId.NO_RULE)) { - // the capture requires additional matching - final var scopeName = captureRule.getName(lineTextContent, captureIndices); - final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar); - final var contentName = captureRule.getContentName(lineTextContent, captureIndices); - final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); - - // the capture requires additional matching - final var stackClone = stack.push(retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList, - contentNameScopesList); - final var onigSubStr = OnigString.of(lineTextContent.substring(0, captureIndex.end)); - tokenizeString(grammar, onigSubStr, isFirstLine && captureIndex.start == 0, captureIndex.start, stackClone, lineTokens, - false, Duration.ZERO /* no time limit */); - continue; - } - - final var captureRuleScopeName = captureRule.getName(lineTextContent, captureIndices); - if (captureRuleScopeName != null) { - // push - final var base = localStack.isEmpty() ? stack.contentNameScopesList : localStack.getLast().scopes; - final var captureRuleScopesList = castNonNull(base).pushAttributed(captureRuleScopeName, grammar); - localStack.add(new LocalStackElement(captureRuleScopesList, captureIndex.end)); - } - } - - while (!localStack.isEmpty()) { - // pop! - final var lastElem = localStack.removeLast(); - lineTokens.produceFromScopes(lastElem.scopes, lastElem.endPos); - } - } - - /** - * Walk the stack from bottom to top, and check each while condition in this order. - * If any fails, cut off the entire stack above the failed while condition. - * While conditions may also advance the linePosition. - */ - private WhileCheckResult checkWhileConditions(final Grammar grammar, final OnigString lineText, boolean isFirstLine, int linePos, - StateStack stack, final LineTokens lineTokens) { - int anchorPosition = stack.beginRuleCapturedEOL ? 0 : -1; - - final class WhileStack { - final StateStack stack; - final BeginWhileRule rule; - - WhileStack(final StateStack stack, final BeginWhileRule rule) { - this.stack = stack; - this.rule = rule; - } - } - - final var whileRules = new ArrayList(); - for (StateStack node = stack; node != null; node = node.pop()) { - final Rule nodeRule = node.getRule(grammar); - if (nodeRule instanceof final BeginWhileRule beginWhileRule) { - whileRules.add(new WhileStack(node, beginWhileRule)); - } - } - - for (int i = whileRules.size() - 1; i >= 0; i--) { - final var whileRule = whileRules.get(i); - - final var ruleScanner = whileRule.rule.compileWhileAG(whileRule.stack.endRule, isFirstLine, anchorPosition == linePos); - final var r = ruleScanner.scanner.findNextMatch(lineText, linePos); - /* if (LOGGER.isLoggable(TRACE)) { - LOGGER.log(TRACE, " scanning for while rule"); - LOGGER.log(TRACE, debugCompiledRuleToString(ruleScanner)); - }*/ - - if (r != null) { - final RuleId matchedRuleId = ruleScanner.rules[r.index]; - if (RuleId.WHILE_RULE.notEquals(matchedRuleId)) { - // we shouldn't end up here - stack = castNonNull(whileRule.stack.pop()); - break; - } - if (r.getCaptureIndices().length > 0) { - lineTokens.produce(whileRule.stack, r.getCaptureIndices()[0].start); - handleCaptures(grammar, lineText, isFirstLine, whileRule.stack, lineTokens, whileRule.rule.whileCaptures, - r.getCaptureIndices()); - lineTokens.produce(whileRule.stack, r.getCaptureIndices()[0].end); - anchorPosition = r.getCaptureIndices()[0].end; - if (r.getCaptureIndices()[0].end > linePos) { - linePos = r.getCaptureIndices()[0].end; - isFirstLine = false; - } - } - } else { - stack = castNonNull(whileRule.stack.pop()); - break; - } - } - - return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine); - } - - static TokenizeStringResult tokenizeString(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, - final int linePos, final StateStack stack, final LineTokens lineTokens, final boolean checkWhileConditions, - final Duration timeLimit) { - return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens) - .scan(checkWhileConditions, timeLimit.toMillis()); - } - - static String debugCompiledRuleToString(final CompiledRule ruleScanner) { - final var r = new ArrayList(ruleScanner.rules.length); - for (int i = 0, l = ruleScanner.rules.length; i < l; i++) { - r.add(" - " + ruleScanner.rules[i] + ": " + ruleScanner.debugRegExps.get(i)); - } - return String.join(System.lineSeparator(), r); - } + bestMatchRating = matchRating; + bestMatchCaptureIndices = matchResult.getCaptureIndices(); + bestMatchRuleId = ruleScanner.rules[matchResult.index]; + bestMatchResultPriority = injection.priority; + + if (bestMatchRating == linePos) { + // No more need to look at the rest of the injections + break; + } + } + + if (bestMatchCaptureIndices != null) { + return new MatchInjectionsResult( + bestMatchRuleId, + bestMatchCaptureIndices, + bestMatchResultPriority == -1); + } + + return null; + } + + private void handleCaptures(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, final StateStack stack, + final LineTokens lineTokens, final List<@Nullable CaptureRule> captures, final OnigCaptureIndex[] captureIndices) { + if (captures.isEmpty()) { + return; + } + + final var lineTextContent = lineText.content; + + final int len = Math.min(captures.size(), captureIndices.length); + final var localStack = new ArrayDeque(); + final int maxEnd = captureIndices[0].end; + + for (int i = 0; i < len; i++) { + final var captureRule = captures.get(i); + if (captureRule == null) { + // Not interested + continue; + } + + final var captureIndex = captureIndices[i]; + + if (captureIndex.getLength() == 0) { + // Nothing really captured + continue; + } + + if (captureIndex.start > maxEnd) { + // Capture going beyond consumed string + break; + } + + // pop captures while needed + while (!localStack.isEmpty() && localStack.getLast().endPos <= captureIndex.start) { + // pop! + final var lastElem = localStack.removeLast(); + lineTokens.produceFromScopes(lastElem.scopes, lastElem.endPos); + } + + if (!localStack.isEmpty()) { + lineTokens.produceFromScopes(localStack.getLast().scopes, captureIndex.start); + } else { + lineTokens.produce(stack, captureIndex.start); + } + + final var retokenizeCapturedWithRuleId = captureRule.retokenizeCapturedWithRuleId; + if (retokenizeCapturedWithRuleId.notEquals(RuleId.NO_RULE)) { + // the capture requires additional matching + final var scopeName = captureRule.getName(lineTextContent, captureIndices); + final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar); + final var contentName = captureRule.getContentName(lineTextContent, captureIndices); + final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); + + // the capture requires additional matching + final var stackClone = stack.push(retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList, + contentNameScopesList); + final var onigSubStr = OnigString.of(lineTextContent.substring(0, captureIndex.end)); + tokenizeString(grammar, onigSubStr, isFirstLine && captureIndex.start == 0, captureIndex.start, stackClone, lineTokens, + false, Duration.ZERO /* no time limit */); + continue; + } + + final var captureRuleScopeName = captureRule.getName(lineTextContent, captureIndices); + if (captureRuleScopeName != null) { + // push + final var base = localStack.isEmpty() ? stack.contentNameScopesList : localStack.getLast().scopes; + final var captureRuleScopesList = castNonNull(base).pushAttributed(captureRuleScopeName, grammar); + localStack.add(new LocalStackElement(captureRuleScopesList, captureIndex.end)); + } + } + + while (!localStack.isEmpty()) { + // pop! + final var lastElem = localStack.removeLast(); + lineTokens.produceFromScopes(lastElem.scopes, lastElem.endPos); + } + } + + /** + * Walk the stack from bottom to top, and check each while condition in this order. + * If any fails, cut off the entire stack above the failed while condition. + * While conditions may also advance the linePosition. + */ + private WhileCheckResult checkWhileConditions(final Grammar grammar, final OnigString lineText, boolean isFirstLine, int linePos, + StateStack stack, final LineTokens lineTokens) { + int anchorPosition = stack.beginRuleCapturedEOL ? 0 : -1; + + final class WhileStack { + final StateStack stack; + final BeginWhileRule rule; + + WhileStack(final StateStack stack, final BeginWhileRule rule) { + this.stack = stack; + this.rule = rule; + } + } + + final var whileRules = new ArrayList(); + for (StateStack node = stack; node != null; node = node.pop()) { + final Rule nodeRule = node.getRule(grammar); + if (nodeRule instanceof final BeginWhileRule beginWhileRule) { + whileRules.add(new WhileStack(node, beginWhileRule)); + } + } + + for (int i = whileRules.size() - 1; i >= 0; i--) { + final var whileRule = whileRules.get(i); + + final var ruleScanner = whileRule.rule.compileWhileAG(whileRule.stack.endRule, isFirstLine, anchorPosition == linePos); + final var r = ruleScanner.scanner.findNextMatch(lineText, linePos); + /*if (LOGGER.isLoggable(TRACE)) { + LOGGER.log(TRACE, " scanning for while rule"); + LOGGER.log(TRACE, debugCompiledRuleToString(ruleScanner)); + }*/ + + if (r != null) { + final RuleId matchedRuleId = ruleScanner.rules[r.index]; + if (RuleId.WHILE_RULE.notEquals(matchedRuleId)) { + // we shouldn't end up here + stack = castNonNull(whileRule.stack.pop()); + break; + } + if (r.getCaptureIndices().length > 0) { + lineTokens.produce(whileRule.stack, r.getCaptureIndices()[0].start); + handleCaptures(grammar, lineText, isFirstLine, whileRule.stack, lineTokens, whileRule.rule.whileCaptures, + r.getCaptureIndices()); + lineTokens.produce(whileRule.stack, r.getCaptureIndices()[0].end); + anchorPosition = r.getCaptureIndices()[0].end; + if (r.getCaptureIndices()[0].end > linePos) { + linePos = r.getCaptureIndices()[0].end; + isFirstLine = false; + } + } + } else { + stack = castNonNull(whileRule.stack.pop()); + break; + } + } + + return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine); + } + + static TokenizeStringResult tokenizeString(final Grammar grammar, final OnigString lineText, final boolean isFirstLine, + final int linePos, final StateStack stack, final LineTokens lineTokens, final boolean checkWhileConditions, + final Duration timeLimit) { + return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens) + .scan(checkWhileConditions, timeLimit.toMillis()); + } + + static String debugCompiledRuleToString(final CompiledRule ruleScanner) { + final var r = new ArrayList(ruleScanner.rules.length); + for (int i = 0, l = ruleScanner.rules.length; i < l; i++) { + r.add(" - " + ruleScanner.rules[i] + ": " + ruleScanner.debugRegExps.get(i)); + } + return String.join(System.lineSeparator(), r); + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java index fce7c41ba..8d2c818df 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java @@ -3,13 +3,13 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Initial code from https://github.com/microsoft/vscode-textmate/ * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. * Initial license: MIT - *

+ * * Contributors: * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java @@ -32,8 +32,6 @@ import java.util.Deque; import java.util.List; -import io.github.rosemoe.sora.util.Logger; - /** * @see @@ -41,150 +39,148 @@ */ final class LineTokens { - private static final class Token implements IToken { - private int startIndex; - private final int endIndex; - private final List scopes; - - Token(final int startIndex, final int endIndex, final List scopes) { - this.startIndex = startIndex; - this.endIndex = endIndex; - this.scopes = scopes; - } - - @Override - public int getStartIndex() { - return startIndex; - } - - @Override - public void setStartIndex(final int startIndex) { - this.startIndex = startIndex; - } - - @Override - public int getEndIndex() { - return endIndex; - } - - @Override - public List getScopes() { - return scopes; - } - - @Override - public String toString() { - return "{" - + "startIndex: " + startIndex - + ", endIndex: " + endIndex - + ", scopes: " + scopes - + "}"; - } - } - - private static final Logger LOGGER = Logger.instance(LineTokens.class.getName()); + private static final class Token implements IToken { + private int startIndex; + private final int endIndex; + private final List scopes; + + Token(final int startIndex, final int endIndex, final List scopes) { + this.startIndex = startIndex; + this.endIndex = endIndex; + this.scopes = scopes; + } + + @Override + public int getStartIndex() { + return startIndex; + } + + @Override + public void setStartIndex(final int startIndex) { + this.startIndex = startIndex; + } + + @Override + public int getEndIndex() { + return endIndex; + } + + @Override + public List getScopes() { + return scopes; + } + + @Override + public String toString() { + return "{" + + "startIndex: " + startIndex + + ", endIndex: " + endIndex + + ", scopes: " + scopes + + "}"; + } + } private static final Deque EMPTY_DEQUE = new ArrayDeque<>(0); - private final boolean _emitBinaryTokens; - - /** - * defined only if `LOGGER.isLoggable(TRACE)`. - */ - private final String _lineText; - - /** - * used only if `emitBinaryTokens` is false. - */ - private final Deque _tokens; - - /** - * used only if `emitBinaryTokens` is true. - */ - private final List _binaryTokens; - - private int _lastTokenEndIndex = 0; - - private final List _tokenTypeOverrides; - - @Nullable - private final BalancedBracketSelectors balancedBracketSelectors; - - LineTokens(final boolean emitBinaryTokens, - final String lineText, - final List tokenTypeOverrides, - @Nullable final BalancedBracketSelectors balancedBracketSelectors) { - - this._emitBinaryTokens = emitBinaryTokens; - this._tokenTypeOverrides = tokenTypeOverrides; - this._lineText =/* LOGGER.isLoggable(TRACE) ? lineText : */""; // store line only if it's logged - if (this._emitBinaryTokens) { - this._tokens = EMPTY_DEQUE; - this._binaryTokens = new ArrayList<>(); - } else { - this._tokens = new ArrayDeque<>(); - this._binaryTokens = Collections.emptyList(); - } - this.balancedBracketSelectors = balancedBracketSelectors; - } - - void produce(final StateStack stack, final int endIndex) { - this.produceFromScopes(stack.contentNameScopesList, endIndex); - } - - void produceFromScopes(@Nullable final AttributedScopeStack scopesList, final int endIndex) { - if (this._lastTokenEndIndex >= endIndex) { - return; - } - - if (this._emitBinaryTokens) { - int metadata = scopesList != null ? scopesList.tokenAttributes : 0; - var containsBalancedBrackets = false; - final var balancedBracketSelectors = this.balancedBracketSelectors; - if (balancedBracketSelectors != null && balancedBracketSelectors.matchesAlways()) { - containsBalancedBrackets = true; - } - - if (!_tokenTypeOverrides.isEmpty() - || balancedBracketSelectors != null - && !balancedBracketSelectors.matchesAlways() && !balancedBracketSelectors.matchesNever()) { - // Only generate scope array when required to improve performance - final List scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList(); - for (final var tokenType : _tokenTypeOverrides) { - if (tokenType.matcher.matches(scopes)) { - metadata = EncodedTokenAttributes.set( - metadata, - 0, - tokenType.type, // toOptionalTokenType(tokenType.type), - null, - FontStyle.NotSet, - 0, - 0); - } - } - if (balancedBracketSelectors != null) { - containsBalancedBrackets = balancedBracketSelectors.match(scopes); - } - } - - if (containsBalancedBrackets) { - metadata = EncodedTokenAttributes.set( - metadata, - 0, - OptionalStandardTokenType.NotSet, - containsBalancedBrackets, - FontStyle.NotSet, - 0, - 0); - } - - if (!this._binaryTokens.isEmpty() && getLastElement(this._binaryTokens) == metadata) { - // no need to push a token with the same metadata - this._lastTokenEndIndex = endIndex; - return; - } - - /*if (LOGGER.isLoggable(TRACE)) { + private final boolean _emitBinaryTokens; + + /** + * defined only if `LOGGER.isLoggable(TRACE)`. + */ + private final String _lineText; + + /** + * used only if `emitBinaryTokens` is false. + */ + private final Deque _tokens; + + /** + * used only if `emitBinaryTokens` is true. + */ + private final List _binaryTokens; + + private int _lastTokenEndIndex = 0; + + private final List _tokenTypeOverrides; + + @Nullable + private final BalancedBracketSelectors balancedBracketSelectors; + + LineTokens(final boolean emitBinaryTokens, + final String lineText, + final List tokenTypeOverrides, + @Nullable final BalancedBracketSelectors balancedBracketSelectors) { + + this._emitBinaryTokens = emitBinaryTokens; + this._tokenTypeOverrides = tokenTypeOverrides; + this._lineText = /*LOGGER.isLoggable(TRACE) ? lineText :*/ ""; // store line only if it's logged + if (this._emitBinaryTokens) { + this._tokens = EMPTY_DEQUE; + this._binaryTokens = new ArrayList<>(); + } else { + this._tokens = new ArrayDeque<>(); + this._binaryTokens = Collections.emptyList(); + } + this.balancedBracketSelectors = balancedBracketSelectors; + } + + void produce(final StateStack stack, final int endIndex) { + this.produceFromScopes(stack.contentNameScopesList, endIndex); + } + + void produceFromScopes(@Nullable final AttributedScopeStack scopesList, final int endIndex) { + if (this._lastTokenEndIndex >= endIndex) { + return; + } + + if (this._emitBinaryTokens) { + int metadata = scopesList != null ? scopesList.tokenAttributes : 0; + var containsBalancedBrackets = false; + final var balancedBracketSelectors = this.balancedBracketSelectors; + if (balancedBracketSelectors != null && balancedBracketSelectors.matchesAlways()) { + containsBalancedBrackets = true; + } + + if (!_tokenTypeOverrides.isEmpty() + || balancedBracketSelectors != null + && !balancedBracketSelectors.matchesAlways() && !balancedBracketSelectors.matchesNever()) { + // Only generate scope array when required to improve performance + final List scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList(); + for (final var tokenType : _tokenTypeOverrides) { + if (tokenType.matcher.matches(scopes)) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + tokenType.type, // toOptionalTokenType(tokenType.type), + null, + FontStyle.NotSet, + 0, + 0); + } + } + if (balancedBracketSelectors != null) { + containsBalancedBrackets = balancedBracketSelectors.match(scopes); + } + } + + if (containsBalancedBrackets) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + OptionalStandardTokenType.NotSet, + containsBalancedBrackets, + FontStyle.NotSet, + 0, + 0); + } + + if (!this._binaryTokens.isEmpty() && getLastElement(this._binaryTokens) == metadata) { + // no need to push a token with the same metadata + this._lastTokenEndIndex = endIndex; + return; + } + + /* if (LOGGER.isLoggable(TRACE)) { final List scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList(); LOGGER.log(TRACE, " token: |" + this._lineText .substring(this._lastTokenEndIndex >= 0 ? this._lastTokenEndIndex : 0, endIndex) @@ -195,14 +191,14 @@ void produceFromScopes(@Nullable final AttributedScopeStack scopesList, final in } }*/ - this._binaryTokens.add(this._lastTokenEndIndex); - this._binaryTokens.add(metadata); + this._binaryTokens.add(this._lastTokenEndIndex); + this._binaryTokens.add(metadata); - this._lastTokenEndIndex = endIndex; - return; - } + this._lastTokenEndIndex = endIndex; + return; + } - final List scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList(); + final List scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList(); /*if (LOGGER.isLoggable(TRACE)) { LOGGER.log(TRACE, " token: |" + this._lineText @@ -214,39 +210,39 @@ void produceFromScopes(@Nullable final AttributedScopeStack scopesList, final in } }*/ - this._tokens.add(new Token(_lastTokenEndIndex, endIndex, scopes)); - - this._lastTokenEndIndex = endIndex; - } - - IToken[] getResult(final StateStack stack, final int lineLength) { - if (!this._tokens.isEmpty() && this._tokens.getLast().getStartIndex() == lineLength - 1) { - // pop produced token for newline - this._tokens.removeLast(); - } - - if (this._tokens.isEmpty()) { - this._lastTokenEndIndex = -1; - this.produce(stack, lineLength); - this._tokens.getLast().setStartIndex(0); - } - - return this._tokens.toArray(IToken[]::new); - } - - int[] getBinaryResult(final StateStack stack, final int lineLength) { - if (!this._binaryTokens.isEmpty() && getElementAt(this._binaryTokens, -2) == lineLength - 1) { - // pop produced token for newline - removeLastElement(this._binaryTokens); - removeLastElement(this._binaryTokens); - } - - if (this._binaryTokens.isEmpty()) { - this._lastTokenEndIndex = -1; - this.produce(stack, lineLength); - this._binaryTokens.set(_binaryTokens.size() - 2, 0); - } - - return _binaryTokens.stream().mapToInt(Integer::intValue).toArray(); - } + this._tokens.add(new Token(_lastTokenEndIndex, endIndex, scopes)); + + this._lastTokenEndIndex = endIndex; + } + + IToken[] getResult(final StateStack stack, final int lineLength) { + if (!this._tokens.isEmpty() && this._tokens.getLast().getStartIndex() == lineLength - 1) { + // pop produced token for newline + this._tokens.removeLast(); + } + + if (this._tokens.isEmpty()) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._tokens.getLast().setStartIndex(0); + } + + return this._tokens.toArray(IToken[]::new); + } + + int[] getBinaryResult(final StateStack stack, final int lineLength) { + if (!this._binaryTokens.isEmpty() && getElementAt(this._binaryTokens, -2) == lineLength - 1) { + // pop produced token for newline + removeLastElement(this._binaryTokens); + removeLastElement(this._binaryTokens); + } + + if (this._binaryTokens.isEmpty()) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._binaryTokens.set(_binaryTokens.size() - 2, 0); + } + + return _binaryTokens.stream().mapToInt(Integer::intValue).toArray(); + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammarReader.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammarReader.java index e37fe843a..03c7c6b37 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammarReader.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammarReader.java @@ -53,7 +53,7 @@ public static RawGrammar readGrammar(final IGrammarSource source) throws Excepti try (var reader = source.getReader()) { return switch (source.getContentType()) { case JSON -> TMParserJSON.INSTANCE.parse(reader, OBJECT_FACTORY); - case YAML -> TMParserYAML.INSTANCE.parse(reader, OBJECT_FACTORY); + case YAML -> TMParserYAML.INSTANCE.parse(reader, OBJECT_FACTORY); default -> TMParserPList.INSTANCE.parse(reader, OBJECT_FACTORY); }; } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/matcher/MatcherBuilder.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/matcher/MatcherBuilder.java index b7bf3535a..137dd6336 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/matcher/MatcherBuilder.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/matcher/MatcherBuilder.java @@ -16,12 +16,13 @@ */ package org.eclipse.tm4e.core.internal.matcher; -import org.eclipse.jdt.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; +import org.eclipse.jdt.annotation.Nullable; + import io.github.rosemoe.sora.util.Logger; /** diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigCaptureIndex.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigCaptureIndex.java index f74d4bf73..36d761e69 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigCaptureIndex.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigCaptureIndex.java @@ -30,8 +30,8 @@ public final class OnigCaptureIndex { public final int end; OnigCaptureIndex(final int start, final int end) { - this.start = Math.max(start, 0); - this.end = Math.max(end, 0); + this.start = start >= 0 ? start : 0; + this.end = end >= 0 ? end : 0; } @Override diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java index 22a278434..76b02c26c 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java @@ -18,8 +18,12 @@ */ package org.eclipse.tm4e.core.internal.oniguruma; + +import java.nio.charset.StandardCharsets; + import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.TMException; +import org.eclipse.tm4e.core.internal.utils.StringUtils; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; import org.joni.Option; @@ -29,8 +33,6 @@ import org.joni.WarnCallback; import org.joni.exception.SyntaxException; -import java.nio.charset.StandardCharsets; - import io.github.rosemoe.sora.util.Logger; /** @@ -38,9 +40,8 @@ * github.com/atom/node-oniguruma/blob/master/src/onig-reg-exp.cc */ public final class OnigRegExp { - private static final Logger LOGGER = Logger.instance(OnigRegExp.class.getName()); - @Nullable + @Nullable private OnigString lastSearchString; private int lastSearchPosition = -1; @@ -48,23 +49,40 @@ public final class OnigRegExp { @Nullable private OnigResult lastSearchResult; + private final String pattern; private final Regex regex; private final boolean hasGAnchor; - public OnigRegExp(final String source) { - hasGAnchor = source.contains("\\G"); - final byte[] pattern = source.getBytes(StandardCharsets.UTF_8); + /** + * @throws TMException if parsing fails + */ + public OnigRegExp(final String pattern) { + this(pattern, false); + } + + /** + * @throws TMException if parsing fails + */ + public OnigRegExp(final String pattern, final boolean ignoreCase) { + this.pattern = pattern; + hasGAnchor = pattern.contains("\\G"); + final byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); try { - regex = new Regex(pattern, 0, pattern.length, Option.CAPTURE_GROUP, UTF8Encoding.INSTANCE, Syntax.DEFAULT, - WarnCallback.DEFAULT); + int options = Option.CAPTURE_GROUP; + if (ignoreCase) + options |= Option.IGNORECASE; + regex = new Regex(patternBytes, 0, patternBytes.length, options, UTF8Encoding.INSTANCE, Syntax.DEFAULT, + /*LOGGER.isLoggable(Level.WARNING) ? LOGGER_WARN_CALLBACK :*/ WarnCallback.NONE); } catch (final SyntaxException ex) { - throw new TMException("Parsing regex pattern \"" + source + "\" failed with " + ex, ex); + throw new TMException("Parsing regex pattern \"" + pattern + "\" failed with " + ex, ex); } } - @Nullable - public OnigResult search(final OnigString str, final int startPosition) { + /** + * @return null if not found + */ + public @Nullable OnigResult search(final OnigString str, final int startPosition) { if (hasGAnchor) { // Should not use caching, because the regular expression // targets the current search position (\G) @@ -78,14 +96,10 @@ public OnigResult search(final OnigString str, final int startPosition) { return lastSearchResult0; } - // multi-thread, lock result - var result = search(str.bytesUTF8, startPosition, str.bytesCount); - synchronized (this) { - lastSearchString = str; - lastSearchPosition = startPosition; - lastSearchResult = result; - } - return result; + lastSearchString = str; + lastSearchPosition = startPosition; + lastSearchResult = search(str.bytesUTF8, startPosition, str.bytesCount); + return lastSearchResult; } @Nullable @@ -98,4 +112,15 @@ private OnigResult search(final byte[] data, final int startPosition, final int } return null; } + + public String pattern() { + return pattern; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> { + sb.append("pattern=").append(pattern); + }); + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigResult.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigResult.java index 1fe0fe5e3..aeac45e88 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigResult.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigResult.java @@ -32,7 +32,7 @@ public final class OnigResult { this.indexInScanner = indexInScanner; } - int getIndex() { + public int getIndex() { return indexInScanner; } @@ -40,17 +40,22 @@ void setIndex(final int index) { indexInScanner = index; } - int locationAt(final int index) { + public int locationAt(final int index) { final int bytes = region.getBeg(index); - return Math.max(bytes, 0); - } + return bytes > 0 ? bytes : 0; + } public int count() { return region.getNumRegs(); } - int lengthAt(final int index) { + public int lengthAt(final int index) { final int bytes = region.getEnd(index) - region.getBeg(index); - return Math.max(bytes, 0); - } + return bytes > 0 ? bytes : 0; + } + + @Override + public String toString() { + return "OnigResult [indexInScanner=" + indexInScanner + ", region=" + region + "]"; + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java index fcef23ab6..217a02774 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java @@ -1,12 +1,12 @@ /** * Copyright (c) 2022 Sebastian Thomschke and others. - *

+ * * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Contributors: * Sebastian Thomschke - initial implementation */ @@ -14,33 +14,29 @@ public interface PropertySettable { - public class ArrayList extends java.util.ArrayList implements PropertySettable { + public class ArrayList extends java.util.ArrayList implements PropertySettable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - @Override - public void setProperty(final String name, final T value) { - final var idx = Integer.parseInt(name); - if (idx == size()) - add(value); - else - set(idx, value); - } - } + @Override + public void setProperty(final String name, final T value) { + final var idx = Integer.parseInt(name); + if (idx == size()) + add(value); + else + set(idx, value); + } + } - public class HashMap extends java.util.HashMap implements PropertySettable { + public class HashMap extends java.util.HashMap implements PropertySettable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - @Override - public void setProperty(final String name, final T value) { - put(name, value); - } - } + @Override + public void setProperty(final String name, final T value) { + put(name, value); + } + } - public interface Factory { - PropertySettable create(I args); - } - - void setProperty(String name, V value); + void setProperty(String name, V value); } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java index edd3cedbe..ea95ad1f1 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java @@ -4,9 +4,9 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Contributors: * - Angelo Zerr - initial API and implementation, see * https://github.com/eclipse/tm4e/blob/95c17ade86677f3e2fd32e76222f71adfce18371/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PList.java @@ -20,7 +20,6 @@ import org.eclipse.jdt.annotation.Nullable; import org.xml.sax.Attributes; import org.xml.sax.InputSource; -import org.xml.sax.SAXException; import org.xml.sax.XMLReader; import org.xml.sax.helpers.DefaultHandler; @@ -32,6 +31,7 @@ import java.util.List; import java.util.Map; +import javax.xml.XMLConstants; import javax.xml.parsers.SAXParserFactory; import io.github.rosemoe.sora.util.Logger; @@ -40,122 +40,121 @@ public final class TMParserPList implements TMParser { private static final Logger LOGGER = Logger.instance(TMParserPList.class.getName()); - private static final String PLIST_ARRAY = "array"; - private static final String PLIST_DICT = "dict"; + private static final String PLIST_ARRAY = "array"; + private static final String PLIST_DICT = "dict"; - public static final TMParserPList INSTANCE = new TMParserPList(); + public static final TMParserPList INSTANCE = new TMParserPList(); - private static final class ParentRef { - final String sourceKind; - final PropertySettable parent; - @Nullable - Object nextPropertyToSet; + private static final class ParentRef { + final String sourceKind; + final PropertySettable parent; + @Nullable + Object nextPropertyToSet; - ParentRef(final String sourceKind, final PropertySettable parent) { - this.sourceKind = sourceKind; - this.parent = parent; - } - } + ParentRef(final String sourceKind, final PropertySettable parent) { + this.sourceKind = sourceKind; + this.parent = parent; + } + } - @Override - public > T parse(final Reader source, final ObjectFactory factory) - throws Exception { - final var spf = SAXParserFactory.newInstance(); - spf.setNamespaceAware(true); + @Override + public > T parse(final Reader source, final ObjectFactory factory) + throws Exception { + final var spf = SAXParserFactory.newInstance(); + spf.setNamespaceAware(true); + // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 + spf.setFeature("http://xml.org/sax/features/external-general-entities", false); + spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 - spf.setFeature("http://xml.org/sax/features/external-general-entities", false); - spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - - final var saxParser = spf.newSAXParser(); + final var saxParser = spf.newSAXParser(); // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 // but not support in android /*saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");*/ - final XMLReader xmlReader = saxParser.getXMLReader(); - xmlReader.setEntityResolver((publicId, systemId) -> new InputSource( - new ByteArrayInputStream("".getBytes()))); - - final T root = factory.createRoot(); - - xmlReader.setContentHandler(new DefaultHandler() { - - final List parents = new ArrayList<>(); - final TMParserPropertyPath path = new TMParserPropertyPath(); - - /** captures the text content of an XML node */ - final StringBuilder text = new StringBuilder(); - - @Override - public void characters(final char @Nullable [] chars, final int start, final int count) { - text.append(chars, start, count); - } - - @Override - @NonNullByDefault({}) - public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) { - text.setLength(0); - switch (localName) { - case PLIST_DICT: { - if (parents.isEmpty()) { - parents.add(new ParentRef(localName, root)); - return; - } - parents.add(new ParentRef(localName, factory.createChild(path, Map.class))); - break; - } - - case PLIST_ARRAY: { - final var newParentRef = new ParentRef(localName, factory.createChild(path, List.class)); - parents.add(newParentRef); - - newParentRef.nextPropertyToSet = 0; - path.add(newParentRef.nextPropertyToSet); - break; - } - } - } - - @Override - @NonNullByDefault({}) - public void endElement(final String uri, final String localName, final String qName) { - switch (localName) { - case PLIST_ARRAY: { - final var parentRef = parents.remove(parents.size() - 1); - - path.removeLast(); // removes the remaining array index from the path - setCurrentProperty(parentRef.parent); // register the constructed object with it's parent - break; - } - - case PLIST_DICT: { - final var parentRef = parents.remove(parents.size() - 1); - - if (!parents.isEmpty()) - setCurrentProperty(parentRef.parent); // register the constructed object with it's parent - break; - } - - case "key": { - final var parentRef = parents.get(parents.size() - 1); + final XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setEntityResolver((publicId, systemId) -> new InputSource( + new ByteArrayInputStream("".getBytes()))); + + final T root = factory.createRoot(); + + xmlReader.setContentHandler(new DefaultHandler() { + + final List parents = new ArrayList<>(); + final TMParserPropertyPath path = new TMParserPropertyPath(); + + /** captures the text content of an XML node */ + final StringBuilder text = new StringBuilder(); + + @Override + public void characters(final char @Nullable [] chars, final int start, final int count) { + text.append(chars, start, count); + } + + @Override + @NonNullByDefault({}) + public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) { + text.setLength(0); + switch (localName) { + case PLIST_DICT: { + if (parents.isEmpty()) { + parents.add(new ParentRef(localName, root)); + return; + } + parents.add(new ParentRef(localName, factory.createChild(path, Map.class))); + break; + } + + case PLIST_ARRAY: { + final var newParentRef = new ParentRef(localName, factory.createChild(path, List.class)); + parents.add(newParentRef); + + newParentRef.nextPropertyToSet = 0; + path.add(newParentRef.nextPropertyToSet); + break; + } + } + } + + @Override + @NonNullByDefault({}) + public void endElement(final String uri, final String localName, final String qName) { + switch (localName) { + case PLIST_ARRAY: { + final var parentRef = parents.remove(parents.size() - 1); + + path.removeLast(); // removes the remaining array index from the path + setCurrentProperty(parentRef.parent); // register the constructed object with it's parent + break; + } + + case PLIST_DICT: { + final var parentRef = parents.remove(parents.size() - 1); + + if (!parents.isEmpty()) + setCurrentProperty(parentRef.parent); // register the constructed object with it's parent + break; + } + + case "key": { + final var parentRef = parents.get(parents.size() - 1); if (!PLIST_DICT.equals(parentRef.sourceKind)) { LOGGER.e(" tag can only be used inside an open element"); break; } - final String key = text.toString(); - parentRef.nextPropertyToSet = key; - path.add(key); - break; - } + final String key = text.toString(); + parentRef.nextPropertyToSet = key; + path.add(key); + break; + } - case "data", "string": - setCurrentProperty(text.toString()); - break; + case "data", "string": + setCurrentProperty(text.toString()); + break; case "date": // e.g. 2007-10-25T12:36:35Z try { @@ -181,43 +180,43 @@ public void endElement(final String uri, final String localName, final String qN } break; - case "true": - setCurrentProperty(Boolean.TRUE); - break; + case "true": + setCurrentProperty(Boolean.TRUE); + break; - case "false": - setCurrentProperty(Boolean.FALSE); - break; + case "false": + setCurrentProperty(Boolean.FALSE); + break; - case "plist": - // ignore - break; + case "plist": + // ignore + break; default: LOGGER.e("Invalid tag name: " + localName); } } - @SuppressWarnings("unchecked") - private void setCurrentProperty(final Object value) { - path.removeLast(); - final var obj = parents.get(parents.size() - 1); - switch (obj.sourceKind) { - case PLIST_ARRAY: - final var idx = castNonNull((Integer) obj.nextPropertyToSet); - ((PropertySettable) obj.parent).setProperty(idx.toString(), value); - obj.nextPropertyToSet = idx + 1; - path.add(obj.nextPropertyToSet); - break; - case PLIST_DICT: - ((PropertySettable) obj.parent).setProperty(castNonNull(obj.nextPropertyToSet).toString(), value); - break; - } - } - }); - - xmlReader.parse(new InputSource(source)); - - return root; - } + @SuppressWarnings("unchecked") + private void setCurrentProperty(final Object value) { + path.removeLast(); + final var obj = parents.get(parents.size() - 1); + switch (obj.sourceKind) { + case PLIST_ARRAY: + final var idx = castNonNull((Integer) obj.nextPropertyToSet); + ((PropertySettable) obj.parent).setProperty(idx.toString(), value); + obj.nextPropertyToSet = idx + 1; + path.add(obj.nextPropertyToSet); + break; + case PLIST_DICT: + ((PropertySettable) obj.parent).setProperty(castNonNull(obj.nextPropertyToSet).toString(), value); + break; + } + } + }); + + xmlReader.parse(new InputSource(source)); + + return root; + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java index f1b666aaa..ab93429b5 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java @@ -62,8 +62,8 @@ public Object last() { return get(size() - 1); } - void removeLast() { - remove(size() - 1); + Object removeLast() { + return remove(size() - 1); } @Override diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java index bdb760930..0d61dbb8c 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java @@ -1,12 +1,12 @@ /** * Copyright (c) 2023 Vegard IT GmbH and others. - *

+ * * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Contributors: * Sebastian Thomschke (Vegard IT) - initial implementation */ @@ -23,17 +23,17 @@ public final class TMParserYAML extends TMParserJSON { - public static final TMParserYAML INSTANCE = new TMParserYAML(); + public static final TMParserYAML INSTANCE = new TMParserYAML(); - private static final LoadSettings LOAD_SETTINGS = LoadSettings.builder() - .setDefaultList(ArrayList::new) - .setDefaultMap(HashMap::new) - .setDefaultSet(HashSet::new) - .build(); + private static final LoadSettings LOAD_SETTINGS = LoadSettings.builder() + .setDefaultList(ArrayList::new) + .setDefaultMap(HashMap::new) + .setDefaultSet(HashSet::new) + .build(); - @Override - @SuppressWarnings({ "null", "unchecked" }) - protected Map loadRaw(final Reader source) { - return (Map) new Load(LOAD_SETTINGS).loadFromReader(source); - } -} \ No newline at end of file + @Override + @SuppressWarnings({ "null", "unchecked" }) + protected Map loadRaw(final Reader source) { + return (Map) new Load(LOAD_SETTINGS).loadFromReader(source); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/registry/SyncRegistry.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/registry/SyncRegistry.java index 3f1ee7b82..f7c84caae 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/registry/SyncRegistry.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/registry/SyncRegistry.java @@ -29,13 +29,14 @@ import org.eclipse.tm4e.core.internal.grammar.raw.IRawGrammar; import org.eclipse.tm4e.core.internal.theme.StyleAttributes; import org.eclipse.tm4e.core.internal.theme.Theme; +import org.eclipse.tm4e.core.internal.utils.ScopeNames; /** * @see * github.com/microsoft/vscode-textmate/blob/main/src/registry.ts */ -public final class SyncRegistry implements IGrammarRepository, IThemeProvider { +public class SyncRegistry implements IGrammarRepository, IThemeProvider { private final Map _grammars = new HashMap<>(); private final Map _rawGrammars = new HashMap<>(); @@ -66,9 +67,18 @@ public void addGrammar(final IRawGrammar grammar, @Nullable final Collection + * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Initial code from https://github.com/microsoft/vscode-textmate/ * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. * Initial license: MIT - *

+ * * Contributors: * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Objects; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,115 +27,90 @@ /** * @see - * github.com/microsoft/vscode-textmate/blob/main/src/rule.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/rule.ts#L582"> + * github.com/microsoft/vscode-textmate/blob/main/src/rule.ts */ final class RegExpSource { - private static final Pattern HAS_BACK_REFERENCES = Pattern.compile("\\\\(\\d+)"); - private static final Pattern BACK_REFERENCING_END = Pattern.compile("\\\\(\\d+)"); - - private String source; - final RuleId ruleId; - final boolean hasBackReferences; - - private String @Nullable [][] anchorCache; - - RegExpSource(final String regExpSource, final RuleId ruleId) { - this(regExpSource, ruleId, true); - } - - RegExpSource(final String regExpSource, final RuleId ruleId, final boolean handleAnchors) { - if (handleAnchors && !regExpSource.isEmpty()) { - final int len = regExpSource.length(); - int lastPushedPos = 0; - final var output = new StringBuilder(); - - boolean hasAnchors = false; - for (int pos = 0; pos < len; pos++) { - final char ch = regExpSource.charAt(pos); - - if (ch == '\\') { - if (pos + 1 < len) { - final char nextCh = regExpSource.charAt(pos + 1); - if (nextCh == 'z') { - output.append(regExpSource.substring(lastPushedPos, pos)); - output.append("$(?!\\n)(?(captureIndices.length); - for (final var capture : captureIndices) { - capturedValues.add(lineText.subSequence(capture.start, capture.end).toString()); - } - - // break change: use lower version of syntax to implement the same feature - - Function matchFunction = match -> { - try { - final int index = Integer.parseInt(match.group(1)); - if (index < captureIndices.length) { - final var replacement = RegexSource.escapeRegExpCharacters(capturedValues.get(index)); - return Matcher.quoteReplacement(replacement); // see https://stackoverflow.com/a/70785772/5116073 - } - } catch (final NumberFormatException ex) { - // ignore - } - return ""; - }; - - var matcher = BACK_REFERENCING_END.matcher(this.source); - var result = new StringBuffer(); - while (matcher.find()) { - matcher.appendReplacement(result, matchFunction.apply(matcher)); - } - matcher.appendTail(result); - return result.toString(); - - /*return BACK_REFERENCING_END.matcher(this.source) - .replaceAll(match -> { + private static final Pattern HAS_BACK_REFERENCES = Pattern.compile("\\\\(\\d+)"); + private static final Pattern BACK_REFERENCING_END = Pattern.compile("\\\\(\\d+)"); + + private String source; + final RuleId ruleId; + final boolean hasBackReferences; + + private String @Nullable [][] anchorCache; + + RegExpSource(final String regExpSource, final RuleId ruleId) { + this(regExpSource, ruleId, true); + } + + RegExpSource(final String regExpSource, final RuleId ruleId, final boolean handleAnchors) { + if (handleAnchors && !regExpSource.isEmpty()) { + final int len = regExpSource.length(); + int lastPushedPos = 0; + final var output = new StringBuilder(); + + boolean hasAnchors = false; + for (int pos = 0; pos < len; pos++) { + final char ch = regExpSource.charAt(pos); + + if (ch == '\\') { + if (pos + 1 < len) { + final char nextCh = regExpSource.charAt(pos + 1); + if (nextCh == 'z') { + output.append(regExpSource.substring(lastPushedPos, pos)); + output.append("$(?!\\n)(?(captureIndices.length); + for (final var capture : captureIndices) { + capturedValues.add(lineText.subSequence(capture.start, capture.end).toString()); + } + return BACK_REFERENCING_END.matcher(this.source).replaceAll(match -> { try { final int index = Integer.parseInt(match.group(1)); if (index < captureIndices.length) { @@ -147,67 +121,67 @@ String resolveBackReferences(final CharSequence lineText, final OnigCaptureIndex // ignore } return ""; - });*/ - } - - private String[][] buildAnchorCache() { - final var source = this.source; - final var sourceLen = source.length(); - - final var A0_G0_result = new StringBuilder(sourceLen); - final var A0_G1_result = new StringBuilder(sourceLen); - final var A1_G0_result = new StringBuilder(sourceLen); - final var A1_G1_result = new StringBuilder(sourceLen); - - for (int pos = 0; pos < sourceLen; pos++) { - final char ch = source.charAt(pos); - A0_G0_result.append(ch); - A0_G1_result.append(ch); - A1_G0_result.append(ch); - A1_G1_result.append(ch); - - if (ch == '\\' && pos + 1 < sourceLen) { - final char nextCh = source.charAt(pos + 1); - if (nextCh == 'A') { - A0_G0_result.append('\uFFFF'); - A0_G1_result.append('\uFFFF'); - A1_G0_result.append('A'); - A1_G1_result.append('A'); - } else if (nextCh == 'G') { - A0_G0_result.append('\uFFFF'); - A0_G1_result.append('G'); - A1_G0_result.append('\uFFFF'); - A1_G1_result.append('G'); - } else { - A0_G0_result.append(nextCh); - A0_G1_result.append(nextCh); - A1_G0_result.append(nextCh); - A1_G1_result.append(nextCh); - } - pos++; - } - } - - return new String[][]{ - {A0_G0_result.toString(), A0_G1_result.toString()}, - {A1_G0_result.toString(), A1_G1_result.toString()} - }; - } - - String resolveAnchors(final boolean allowA, final boolean allowG) { - final var anchorCache = this.anchorCache; - if (anchorCache == null) { - return this.source; - } - - return anchorCache[allowA ? 1 : 0][allowG ? 1 : 0]; - } - - boolean hasAnchor() { - return anchorCache != null; - } - - String getSource() { - return this.source; - } + }); + } + + private String[][] buildAnchorCache() { + final var source = this.source; + final var sourceLen = source.length(); + + final var A0_G0_result = new StringBuilder(sourceLen); + final var A0_G1_result = new StringBuilder(sourceLen); + final var A1_G0_result = new StringBuilder(sourceLen); + final var A1_G1_result = new StringBuilder(sourceLen); + + for (int pos = 0, len = sourceLen; pos < len; pos++) { + final char ch = source.charAt(pos); + A0_G0_result.append(ch); + A0_G1_result.append(ch); + A1_G0_result.append(ch); + A1_G1_result.append(ch); + + if (ch == '\\' && pos + 1 < len) { + final char nextCh = source.charAt(pos + 1); + if (nextCh == 'A') { + A0_G0_result.append('\uFFFF'); + A0_G1_result.append('\uFFFF'); + A1_G0_result.append('A'); + A1_G1_result.append('A'); + } else if (nextCh == 'G') { + A0_G0_result.append('\uFFFF'); + A0_G1_result.append('G'); + A1_G0_result.append('\uFFFF'); + A1_G1_result.append('G'); + } else { + A0_G0_result.append(nextCh); + A0_G1_result.append(nextCh); + A1_G0_result.append(nextCh); + A1_G1_result.append(nextCh); + } + pos++; + } + } + + return new String[][] { + { A0_G0_result.toString(), A0_G1_result.toString() }, + { A1_G0_result.toString(), A1_G1_result.toString() } + }; + } + + String resolveAnchors(final boolean allowA, final boolean allowG) { + final var anchorCache = this.anchorCache; + if (anchorCache == null) { + return this.source; + } + + return anchorCache[allowA ? 1 : 0][allowG ? 1 : 0]; + } + + boolean hasAnchor() { + return anchorCache != null; + } + + String getSource() { + return this.source; + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/rule/RuleFactory.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/rule/RuleFactory.java index de11d517c..3641ea74d 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/rule/RuleFactory.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/rule/RuleFactory.java @@ -16,8 +16,14 @@ */ package org.eclipse.tm4e.core.internal.rule; -import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull; -import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.defaultIfNull; + +import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.*; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.internal.grammar.dependencies.IncludeReference; @@ -26,17 +32,12 @@ import org.eclipse.tm4e.core.internal.grammar.raw.IRawRule; import org.eclipse.tm4e.core.internal.grammar.raw.RawRule; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - import io.github.rosemoe.sora.util.Logger; /** * @see - * github.com/microsoft/vscode-textmate/blob/main/src/rule.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/rule.ts#L381"> + * github.com/microsoft/vscode-textmate/blob/main/src/rule.ts */ public final class RuleFactory { diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/ColorMap.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/ColorMap.java index 249588af4..92cd4529f 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/ColorMap.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/ColorMap.java @@ -1,30 +1,16 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2024 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + *

+ * SPDX-License-Identifier: EPL-2.0 + *

+ * Contributors: + * Angelo Zerr - initial API and implementation */ package org.eclipse.tm4e.core.internal.theme; -import android.util.Log; - import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -35,8 +21,8 @@ /** * @see - * github.com/microsoft/vscode-textmate/blob/main/src/theme.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/theme.ts#L385"> + * github.com/microsoft/vscode-textmate/blob/main/src/theme.ts */ public final class ColorMap { @@ -63,10 +49,10 @@ public ColorMap(@Nullable final List _colorMap) { // old version method public String getColor(int id) { - Log.e("color map", String.format("id2color %s, color2id %s", _id2color, _color2id)); return _id2color.get(id); } + public int getId(@Nullable final String _color) { if (_color == null) { return 0; diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/FontStyle.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/FontStyle.java index 46e37483b..30e663d08 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/FontStyle.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/FontStyle.java @@ -1,25 +1,13 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2024 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + *

+ * SPDX-License-Identifier: EPL-2.0 + *

+ * Contributors: + * Angelo Zerr - initial API and implementation */ package org.eclipse.tm4e.core.internal.theme; @@ -27,8 +15,8 @@ * Font style definitions. * * @see - * https://github.com/microsoft/vscode-textmate/blob/main/src/theme.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/theme.ts#L306"> + * https://github.com/microsoft/vscode-textmate/blob/main/src/theme.ts */ public final class FontStyle { @@ -62,7 +50,7 @@ public static String fontStyleToString(final int fontStyle) { if ((fontStyle & Strikethrough) == Strikethrough) { style.append("strikethrough "); } - // style.isEmpty() no available in android + // String.isEmpty() no available in android if (style.length() < 1) { return "none"; } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/Theme.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/Theme.java index 4f70c82ac..e565389af 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/Theme.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/theme/Theme.java @@ -1,25 +1,13 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2024 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA + * SPDX-License-Identifier: EPL-2.0 * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions + * Contributors: + * Angelo Zerr - initial API and implementation */ package org.eclipse.tm4e.core.internal.theme; @@ -45,94 +33,94 @@ * TextMate theme. * * @see - * github.com/microsoft/vscode-textmate/blob/main/src/theme.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/theme.ts#L7"> + * github.com/microsoft/vscode-textmate/blob/main/src/theme.ts */ public final class Theme { - public static Theme createFromRawTheme(@Nullable final IRawTheme source, @Nullable final List colorMap) { - return createFromParsedTheme(parseTheme(source), colorMap); - } - - public static Theme createFromParsedTheme(final List source, @Nullable final List colorMap) { - return resolveParsedThemeRules(source, colorMap); - } - - private final Map> _cachedMatchRoot = new HashMap<>(); - - private final ColorMap _colorMap; - private final StyleAttributes _defaults; - private final ThemeTrieElement _root; - - public Theme(final ColorMap colorMap, final StyleAttributes defaults, final ThemeTrieElement root) { - this._colorMap = colorMap; - this._root = root; - this._defaults = defaults; - } - - public List getColorMap() { - return this._colorMap.getColorMap(); - } - - public StyleAttributes getDefaults() { - return this._defaults; - } - - @Nullable - public StyleAttributes match(@Nullable final ScopeStack scopePath) { - if (scopePath == null) { - return this._defaults; - } - final var scopeName = scopePath.scopeName; - - final var matchingTrieElements = this._cachedMatchRoot.computeIfAbsent(scopeName, _root::match); - - final var effectiveRule = findFirstMatching(matchingTrieElements, - v -> _scopePathMatchesParentScopes(scopePath.parent, v.parentScopes)); - if (effectiveRule == null) { - return null; - } - - return StyleAttributes.of( - effectiveRule.fontStyle, - effectiveRule.foreground, - effectiveRule.background); - } - - private boolean _scopePathMatchesParentScopes(@Nullable ScopeStack scopePath, @Nullable final List parentScopeNames) { - if (parentScopeNames == null) { - return true; - } - - var index = 0; - var scopePattern = parentScopeNames.get(index); - - while (scopePath != null) { - if (_matchesScope(scopePath.scopeName, scopePattern)) { - index++; - if (index == parentScopeNames.size()) { - return true; - } - scopePattern = parentScopeNames.get(index); - } - scopePath = scopePath.parent; - } - - return false; - } - - private boolean _matchesScope(final String scopeName, final String scopeNamePattern) { - return scopeNamePattern.equals(scopeName) - || scopeName.startsWith(scopeNamePattern) && scopeName.charAt(scopeNamePattern.length()) == '.'; - } - - /** - * Parse a raw theme into rules. - */ - public static List parseTheme(@Nullable final IRawTheme source) { - if (source == null) { - return Collections.emptyList(); - } + public static Theme createFromRawTheme(@Nullable final IRawTheme source, @Nullable final List colorMap) { + return createFromParsedTheme(parseTheme(source), colorMap); + } + + public static Theme createFromParsedTheme(final List source, @Nullable final List colorMap) { + return resolveParsedThemeRules(source, colorMap); + } + + private final Map> _cachedMatchRoot = new HashMap<>(); + + private final ColorMap _colorMap; + private final StyleAttributes _defaults; + private final ThemeTrieElement _root; + + public Theme(final ColorMap colorMap, final StyleAttributes defaults, final ThemeTrieElement root) { + this._colorMap = colorMap; + this._root = root; + this._defaults = defaults; + } + + public List getColorMap() { + return this._colorMap.getColorMap(); + } + + public StyleAttributes getDefaults() { + return this._defaults; + } + + @Nullable + public StyleAttributes match(@Nullable final ScopeStack scopePath) { + if (scopePath == null) { + return this._defaults; + } + final var scopeName = scopePath.scopeName; + + final var matchingTrieElements = this._cachedMatchRoot.computeIfAbsent(scopeName, _root::match); + + final var effectiveRule = findFirstMatching(matchingTrieElements, + v -> _scopePathMatchesParentScopes(scopePath.parent, v.parentScopes)); + if (effectiveRule == null) { + return null; + } + + return StyleAttributes.of( + effectiveRule.fontStyle, + effectiveRule.foreground, + effectiveRule.background); + } + + private boolean _scopePathMatchesParentScopes(@Nullable ScopeStack scopePath, @Nullable final List parentScopeNames) { + if (parentScopeNames == null) { + return true; + } + + var index = 0; + var scopePattern = parentScopeNames.get(index); + + while (scopePath != null) { + if (_matchesScope(scopePath.scopeName, scopePattern)) { + index++; + if (index == parentScopeNames.size()) { + return true; + } + scopePattern = parentScopeNames.get(index); + } + scopePath = scopePath.parent; + } + + return false; + } + + private boolean _matchesScope(final String scopeName, final String scopeNamePattern) { + return scopeNamePattern.equals(scopeName) + || scopeName.startsWith(scopeNamePattern) && scopeName.charAt(scopeNamePattern.length()) == '.'; + } + + /** + * Parse a raw theme into rules. + */ + public static List parseTheme(@Nullable final IRawTheme source) { + if (source == null) { + return Collections.emptyList(); + } var settings = source.getSettings(); @@ -145,106 +133,107 @@ public static List parseTheme(@Nullable final IRawTheme source) return Collections.emptyList(); } - - final var result = new ArrayList(); - int i = -1; - for (final IRawThemeSetting entry : settings) { - final var entrySetting = entry.getSetting(); - if (entrySetting == null) { - continue; - } - - i++; - - final Object settingScope = entry.getScope(); - final List scopes; - if (settingScope instanceof String _scope) { - // remove leading commas - _scope = _scope.replaceAll("^,+", ""); - - // remove trailing commas - _scope = _scope.replaceAll(",+$", ""); - - scopes = StringUtils.splitToList(_scope, ','); - } else if (settingScope instanceof List) { - @SuppressWarnings("unchecked") final var settingScopes = (List) settingScope; - scopes = settingScopes; - } else { - scopes = List.of(""); - } - - int fontStyle = FontStyle.NotSet; - final var settingsFontStyle = entrySetting.getFontStyle(); - if (settingsFontStyle instanceof final String style) { - fontStyle = FontStyle.None; - - final var segments = StringUtils.splitToArray(style, ' '); - for (final var segment : segments) { - switch (segment) { - case "italic": - fontStyle = fontStyle | FontStyle.Italic; - break; - case "bold": - fontStyle = fontStyle | FontStyle.Bold; - break; - case "underline": - fontStyle = fontStyle | FontStyle.Underline; - break; - case "strikethrough": - fontStyle = fontStyle | FontStyle.Strikethrough; - break; - } - } - } - - String foreground = null; - final Object settingsForeground = entrySetting.getForeground(); - if (settingsForeground instanceof final String stringSettingsForeground - && isValidHexColor(stringSettingsForeground)) { - foreground = stringSettingsForeground; - } - - String background = null; - final Object settingsBackground = entrySetting.getBackground(); - if (settingsBackground instanceof final String stringSettingsBackground - && isValidHexColor(stringSettingsBackground)) { - background = stringSettingsBackground; - } - - for (int j = 0, lenJ = scopes.size(); j < lenJ; j++) { - final var _scope = scopes.get(j).trim(); - - final var segments = StringUtils.splitToList(_scope, ' '); - - final var scope = getLastElement(segments); - List parentScopes = null; - if (segments.size() > 1) { - parentScopes = new ArrayList<>(segments.subList(0, segments.size() - 1)); - Collections.reverse(parentScopes); - } - - result.add(new ParsedThemeRule( - scope, - parentScopes, - i, - fontStyle, - foreground, - background)); - } - } - - return result; - } - - /** - * Resolve rules (i.e. inheritance). - */ - public static Theme resolveParsedThemeRules(final List _parsedThemeRules, @Nullable final List _colorMap) { - - // copy the list since we cannot be sure the given list is mutable - final var parsedThemeRules = new ArrayList<>(_parsedThemeRules); + final var result = new ArrayList(); + int i = -1; + for (final IRawThemeSetting entry : settings) { + final var entrySetting = entry.getSetting(); + if (entrySetting == null) { + continue; + } + + i++; + + final Object settingScope = entry.getScope(); + final List scopes; + if (settingScope instanceof String _scope) { + // remove leading commas + _scope = _scope.replaceAll("^,+", ""); + + // remove trailing commas + _scope = _scope.replaceAll(",+$", ""); + + scopes = StringUtils.splitToList(_scope, ','); + } else if (settingScope instanceof List) { + @SuppressWarnings("unchecked") + final var settingScopes = (List) settingScope; + scopes = settingScopes; + } else { + scopes = List.of(""); + } + + int fontStyle = FontStyle.NotSet; + final var settingsFontStyle = entrySetting.getFontStyle(); + if (settingsFontStyle instanceof final String style) { + fontStyle = FontStyle.None; + + final var segments = StringUtils.splitToArray(style, ' '); + for (final var segment : segments) { + switch (segment) { + case "italic": + fontStyle = fontStyle | FontStyle.Italic; + break; + case "bold": + fontStyle = fontStyle | FontStyle.Bold; + break; + case "underline": + fontStyle = fontStyle | FontStyle.Underline; + break; + case "strikethrough": + fontStyle = fontStyle | FontStyle.Strikethrough; + break; + } + } + } + + String foreground = null; + final Object settingsForeground = entrySetting.getForeground(); + if (settingsForeground instanceof final String stringSettingsForeground + && isValidHexColor(stringSettingsForeground)) { + foreground = stringSettingsForeground; + } + + String background = null; + final Object settingsBackground = entrySetting.getBackground(); + if (settingsBackground instanceof final String stringSettingsBackground + && isValidHexColor(stringSettingsBackground)) { + background = stringSettingsBackground; + } + + for (int j = 0, lenJ = scopes.size(); j < lenJ; j++) { + final var _scope = scopes.get(j).trim(); + + final var segments = StringUtils.splitToList(_scope, ' '); + + final var scope = getLastElement(segments); + List parentScopes = null; + if (segments.size() > 1) { + parentScopes = new ArrayList<>(segments.subList(0, segments.size() - 1)); + Collections.reverse(parentScopes); + } + + result.add(new ParsedThemeRule( + scope, + parentScopes, + i, + fontStyle, + foreground, + background)); + } + } + + return result; + } + + /** + * Resolve rules (i.e. inheritance). + */ + public static Theme resolveParsedThemeRules(final List _parsedThemeRules, @Nullable final List _colorMap) { + + // copy the list since we cannot be sure the given list is mutable + final var parsedThemeRules = new ArrayList<>(_parsedThemeRules); // Sort rules lexicographically, and then by index if necessary + // Use Collections.sort() to adapted android api 21 Collections.sort(parsedThemeRules, (a, b) -> { int r = strcmp(a.scope, b.scope); if (r != 0) { @@ -257,33 +246,33 @@ public static Theme resolveParsedThemeRules(final List _parsedT return a.index - b.index; }); - // Determine defaults - int defaultFontStyle = FontStyle.None; - String defaultForeground = "#000000"; - String defaultBackground = "#ffffff"; - while (!parsedThemeRules.isEmpty() && parsedThemeRules.get(0).scope.isEmpty()) { - final var incomingDefaults = parsedThemeRules.remove(0); - if (incomingDefaults.fontStyle != FontStyle.NotSet) { - defaultFontStyle = incomingDefaults.fontStyle; - } - if (incomingDefaults.foreground != null) { - defaultForeground = incomingDefaults.foreground; - } - if (incomingDefaults.background != null) { - defaultBackground = incomingDefaults.background; - } - } - final var colorMap = new ColorMap(_colorMap); - final var defaults = StyleAttributes.of(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); - - final var root = new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, 0, 0), Collections.emptyList()); - for (int i = 0, len = parsedThemeRules.size(); i < len; i++) { - final var rule = parsedThemeRules.get(i); - root.insert(0, rule.scope, rule.parentScopes, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background)); - } - - return new Theme(colorMap, defaults, root); - } + // Determine defaults + int defaultFontStyle = FontStyle.None; + String defaultForeground = "#000000"; + String defaultBackground = "#ffffff"; + while (!parsedThemeRules.isEmpty() && parsedThemeRules.get(0).scope.isEmpty()) { + final var incomingDefaults = parsedThemeRules.remove(0); + if (incomingDefaults.fontStyle != FontStyle.NotSet) { + defaultFontStyle = incomingDefaults.fontStyle; + } + if (incomingDefaults.foreground != null) { + defaultForeground = incomingDefaults.foreground; + } + if (incomingDefaults.background != null) { + defaultBackground = incomingDefaults.background; + } + } + final var colorMap = new ColorMap(_colorMap); + final var defaults = StyleAttributes.of(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); + + final var root = new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, 0, 0), Collections.emptyList()); + for (int i = 0, len = parsedThemeRules.size(); i < len; i++) { + final var rule = parsedThemeRules.get(i); + root.insert(0, rule.scope, rule.parentScopes, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background)); + } + + return new Theme(colorMap, defaults, root); + } /** * Used to get the color, inherited from the old version of the method @@ -302,14 +291,14 @@ public int hashCode() { return 31 * result + _root.hashCode(); } - @Override - public boolean equals(@Nullable final Object obj) { - if (this == obj) - return true; - if (obj instanceof final Theme other) - return Objects.equals(_colorMap, other._colorMap) - && Objects.equals(_defaults, other._defaults) - && Objects.equals(_root, other._root); - return false; - } + @Override + public boolean equals(@Nullable final Object obj) { + if (this == obj) + return true; + if (obj instanceof final Theme other) + return Objects.equals(_colorMap, other._colorMap) + && Objects.equals(_defaults, other._defaults) + && Objects.equals(_root, other._root); + return false; + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/AbstractListeners.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/AbstractListeners.java index e945fd4a9..2c9f9c868 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/AbstractListeners.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/AbstractListeners.java @@ -1,12 +1,12 @@ /** * Copyright (c) 2023 Vegard IT GmbH and others. - *

+ * * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Contributors: * Sebastian Thomschke - initial implementation */ @@ -26,43 +26,43 @@ */ public abstract class AbstractListeners { - private final Set listeners = new CopyOnWriteArraySet<>(); + private final Set listeners = new CopyOnWriteArraySet<>(); public boolean add(final LISTENER listener) { // LOGGER.log(DEBUG, "Trying to add listener {0} which is already registered with {1}.", listener, this); return listeners.add(listener); } - public int count() { - return listeners.size(); - } + public int count() { + return listeners.size(); + } - /** - * Forwards the given event to all registered listeners. - */ - public void dispatchEvent(final EVENT e) { - listeners.forEach(l -> dispatchEvent(e, l)); - } + /** + * Forwards the given event to all registered listeners. + */ + public void dispatchEvent(final EVENT e) { + listeners.forEach(l -> dispatchEvent(e, l)); + } - /** - * Forwards the given event to the given listeners. - */ - public abstract void dispatchEvent(EVENT e, LISTENER l); + /** + * Forwards the given event to the given listeners. + */ + public abstract void dispatchEvent(EVENT e, LISTENER l); - public boolean isEmpty() { - return listeners.isEmpty(); - } + public boolean isEmpty() { + return listeners.isEmpty(); + } - public boolean isNotEmpty() { - return !listeners.isEmpty(); - } + public boolean isNotEmpty() { + return !listeners.isEmpty(); + } public boolean remove(final LISTENER listener) { // LOGGER.log(WARNING, "Trying to remove listener {0} which is not registered with {1}.", listener, this); return listeners.remove(listener); } - public void removeAll() { - listeners.clear(); - } + public void removeAll() { + listeners.clear(); + } } \ No newline at end of file diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/MoreCollections.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/MoreCollections.java index 3b6edabb1..6ddd96c99 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/MoreCollections.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/MoreCollections.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Predicate; import org.eclipse.jdt.annotation.Nullable; @@ -81,11 +82,13 @@ public static List nullToEmpty(@Nullable final List list) { /** * Removes the last element in this list. * + * @return the element previously at the specified position + * * @throws UnsupportedOperationException if the {@code remove} operation is not supported by this list - * @throws IndexOutOfBoundsException if the list is empty + * @throws IndexOutOfBoundsException if the list is empty */ - public static void removeLastElement(final List list) { - list.remove(list.size() - 1); + public static T removeLastElement(final List list) { + return list.remove(list.size() - 1); } public static String toStringWithIndex(final List list) { @@ -103,6 +106,13 @@ public static String toStringWithIndex(final List list) { return sb.append(']').toString(); } + /** + * @return a new list without null elements + */ + public static List noNulls(final @Nullable List coll) { + return coll == null || coll.isEmpty() ? Collections.emptyList() : coll.stream().filter(Objects::nonNull).toList(); + } + private MoreCollections() { } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ObjectCloner.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ObjectCloner.java index 1b5e700bd..b1cd4bcce 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ObjectCloner.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ObjectCloner.java @@ -1,10 +1,10 @@ /** * Copyright (c) 2022 Sebastian Thomschke and others. - *

+ * * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 */ package org.eclipse.tm4e.core.internal.utils; @@ -19,28 +19,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.WeakHashMap; import java.util.function.Supplier; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; - public final class ObjectCloner { - private static final LoadingCache, Optional> CLONE_METHODS = CacheBuilder.newBuilder().weakKeys() - .build(new CacheLoader<>() { - @Override - public Optional load(final Class cls) { - try { - return Optional.of(cls.getMethod("clone")); - } catch (final Exception ex) { - return Optional.empty(); - } - } - }); + private static final WeakHashMap, Optional> CLONE_METHODS_CACHE = new WeakHashMap<>(); public static <@NonNull T> T deepClone(final T obj) { return deepClone(obj, new IdentityHashMap<>()); @@ -57,9 +44,7 @@ public Optional load(final Class cls) { var list = (List) obj; final var listClone = shallowClone(list, () -> new ArrayList<>(list)); clones.put(list, listClone); - for (var i = 0; i < listClone.size(); ++i) { - listClone.set(i, deepCloneNullable(listClone.get(i), clones)); - } + listClone.replaceAll(v -> deepCloneNullable(v, clones)); return (T) listClone; } @@ -108,10 +93,15 @@ public Optional load(final Class cls) { @SuppressWarnings("unchecked") private static <@NonNull T> T shallowClone(final T obj, final Supplier fallback) { - final var objClass = obj.getClass(); if (obj instanceof Cloneable) { try { - final var cloneMethod = CLONE_METHODS.get(objClass); + final var cloneMethod = CLONE_METHODS_CACHE.computeIfAbsent(obj.getClass(), cls -> { + try { + return Optional.of(cls.getMethod("clone")); + } catch (final Exception ex) { + return Optional.empty(); + } + }); if (cloneMethod.isPresent()) { return (T) cloneMethod.get().invoke(obj); } @@ -124,4 +114,4 @@ public Optional load(final Class cls) { private ObjectCloner() { } -} \ No newline at end of file +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/RegexSource.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/RegexSource.java index 520a37c14..4a9f05ab7 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/RegexSource.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/RegexSource.java @@ -24,22 +24,22 @@ /** * @see - * github.com/microsoft/vscode-textmate/blob/main/src/utils.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/utils.ts#L59"> + * github.com/microsoft/vscode-textmate/blob/main/src/utils.ts */ public final class RegexSource { /* private static final Pattern CAPTURING_REGEX_SOURCE = Pattern .compile("\\$(\\d+)|\\$\\{(\\d+):\\/(downcase|upcase)}");*/ - //fix for android - private static final Pattern CAPTURING_REGEX_SOURCE = Pattern - .compile("\\$(\\d+)|\\$\\{(\\d+):/(downcase|upcase)\\}"); + //fix for android + private static final Pattern CAPTURING_REGEX_SOURCE = Pattern + .compile("\\$(\\d+)|\\$\\{(\\d+):/(downcase|upcase)\\}"); /** * Escapes/prefixes RegEx meta characters with a backslash in the given string. - *

+ * * It is a non-regex based faster alternative to the TypeScript * implementation: diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ScopeNames.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ScopeNames.java new file mode 100644 index 000000000..9fbccb26e --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/ScopeNames.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2024 Sebastian Thomschke and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * - Sebastian Thomschke - initial API and implementation + */ +package org.eclipse.tm4e.core.internal.utils; + +import org.eclipse.jdt.annotation.Nullable; + +/** + * Utility class to deal with plugin scoped TextMate Scope Names, e.g. "source.batchfile@com.example.myplugin" + */ +public final class ScopeNames { + + public static final char CONTRIBUTOR_SEPARATOR = '@'; + + public static @Nullable String getContributor(final String scopeName) { + final int separatorAt = scopeName.indexOf(CONTRIBUTOR_SEPARATOR); + if (separatorAt == -1) { + return ""; + } + return scopeName.substring(separatorAt + 1); + } + + /** + * @return true if a scope name is suffixed by the contributing plugin id, e.g. "source.batchfile@com.example.myplugin" + */ + public static boolean hasContributor(final String scopeName) { + return scopeName.indexOf(CONTRIBUTOR_SEPARATOR) > -1; + } + + public static String withoutContributor(final String scopeName) { + final int separatorAt = scopeName.indexOf(CONTRIBUTOR_SEPARATOR); + if (separatorAt == -1) { + return scopeName; + } + return scopeName.substring(0, separatorAt); + } + + private ScopeNames() { + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/StringUtils.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/StringUtils.java index c3b4e28ea..2a8cfb3e6 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/StringUtils.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/utils/StringUtils.java @@ -3,13 +3,13 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 - *

+ * * Initial code from https://github.com/microsoft/vscode-textmate/ * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. * Initial license: MIT - *

+ * * Contributors: * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java @@ -31,155 +31,171 @@ */ public final class StringUtils { - private static final List LIST_WITH_EMPTY_STRING = List.of(""); + private static final List LIST_WITH_EMPTY_STRING = List.of(""); - private static final Pattern RRGGBB = Pattern.compile("^#[0-9a-f]{6}", Pattern.CASE_INSENSITIVE); - private static final Pattern RRGGBBAA = Pattern.compile("^#[0-9a-f]{8}", Pattern.CASE_INSENSITIVE); - private static final Pattern RGB = Pattern.compile("^#[0-9a-f]{3}", Pattern.CASE_INSENSITIVE); - private static final Pattern RGBA = Pattern.compile("^#[0-9a-f]{4}", Pattern.CASE_INSENSITIVE); + private static final Pattern RRGGBB = Pattern.compile("^#[0-9a-f]{6}", Pattern.CASE_INSENSITIVE); + private static final Pattern RRGGBBAA = Pattern.compile("^#[0-9a-f]{8}", Pattern.CASE_INSENSITIVE); + private static final Pattern RGB = Pattern.compile("^#[0-9a-f]{3}", Pattern.CASE_INSENSITIVE); + private static final Pattern RGBA = Pattern.compile("^#[0-9a-f]{4}", Pattern.CASE_INSENSITIVE); - public static String nullToEmpty(@Nullable final String txt) { - return txt == null ? "" : txt; - } + public static String nullToEmpty(@Nullable final String txt) { + return txt == null ? "" : txt; + } - public static boolean isNullOrEmpty(@Nullable final String txt) { - return txt == null || txt.isEmpty(); - } + public static boolean isNullOrEmpty(@Nullable final String txt) { + return txt == null || txt.isEmpty(); + } public static boolean isValidHexColor(final CharSequence hex) { if (hex.length() < 1) { return false; } - if (RRGGBB.matcher(hex).matches()) { - // #rrggbb - return true; - } - - if (RRGGBBAA.matcher(hex).matches()) { - // #rrggbbaa - return true; - } - - if (RGB.matcher(hex).matches()) { - // #rgb - return true; - } - - if (RGBA.matcher(hex).matches()) { - // #rgba - return true; - } - - return false; - } - - /** - * Very fast String splitting. - * - * 7.5 times faster than {@link String#split(String)} and 2.5 times faster than {@link com.google.common.base.Splitter}. - */ - public static String[] splitToArray(final String line, final char separator) { - if (line.isEmpty()) - return new String[]{""}; - - var tmp = new String[8]; - int count = 0; - int start = 0; - int end = line.indexOf(separator); - while (end >= 0) { - if (count == tmp.length) { // check if array needs resize - final var tmp2 = new String[tmp.length + (tmp.length >> 1)]; - System.arraycopy(tmp, 0, tmp2, 0, count); - tmp = tmp2; - } - tmp[count] = line.substring(start, end); - count++; - start = end + 1; - end = line.indexOf(separator, start); - } - if (count == tmp.length) { // check if array needs resize - final var tmp2 = new String[tmp.length + 1]; - System.arraycopy(tmp, 0, tmp2, 0, count); - tmp = tmp2; - } - tmp[count] = line.substring(start); - count++; - - if (count == tmp.length) { - return tmp; - } - final var result = new String[count]; - System.arraycopy(tmp, 0, result, 0, count); - return result; - } - - /** - * Very fast String splitting. - */ - public static List splitToList(final String line, final char separator) { - if (line.isEmpty()) - return LIST_WITH_EMPTY_STRING; - - final var result = new ArrayList(8); - int start = 0; - int end = line.indexOf(separator); - while (end >= 0) { - result.add(line.substring(start, end)); - start = end + 1; - end = line.indexOf(separator, start); - } - result.add(line.substring(start)); - return result; - } - - public static int strcmp(final String a, final String b) { - final int result = a.compareTo(b); - if (result < 0) { - return -1; - } else if (result > 0) { - return 1; - } - return 0; - } - - public static int strArrCmp(@Nullable final List a, @Nullable final List b) { - if (a == null && b == null) { - return 0; - } - if (a == null) { - return -1; - } - if (b == null) { - return 1; - } - final int len1 = a.size(); - final int len2 = b.size(); - if (len1 == len2) { - for (int i = 0; i < len1; i++) { - final int res = strcmp(a.get(i), b.get(i)); - if (res != 0) { - return res; - } - } - return 0; - } - return len1 - len2; - } - - /** - * @return "{SimpleClassName}{...fields...}" - */ - public static String toString(@Nullable final Object object, final Consumer fieldsBuilder) { - if (object == null) - return "null"; - final var sb = new StringBuilder(object.getClass().getSimpleName()); - sb.append('{'); - fieldsBuilder.accept(sb); - sb.append('}'); - return sb.toString(); - } - - private StringUtils() { - } + if (RRGGBB.matcher(hex).matches()) { + // #rrggbb + return true; + } + + if (RRGGBBAA.matcher(hex).matches()) { + // #rrggbbaa + return true; + } + + if (RGB.matcher(hex).matches()) { + // #rgb + return true; + } + + if (RGBA.matcher(hex).matches()) { + // #rgba + return true; + } + + return false; + } + + /** + * Very fast String splitting. + * + * 7.5 times faster than {@link String#split(String)} and 2.5 times faster than {@link com.google.common.base.Splitter}. + */ + public static String[] splitToArray(final String line, final char separator) { + return splitToArray(line, separator, -1); + } + + /** + * Very fast String splitting. + * + * 7.5 times faster than {@link String#split(String)} and 2.5 times faster than {@link com.google.common.base.Splitter}. + */ + public static String[] splitToArray(final String line, final char separator, final int limit) { + if (line.isEmpty()) + return new String[] { "" }; + + var tmp = new String[8]; + int count = 0; + int start = 0; + int end = line.indexOf(separator, 0); + while (end >= 0) { + if (count == tmp.length) { // check if array needs resize + final var tmp2 = new String[tmp.length + (tmp.length >> 1)]; + System.arraycopy(tmp, 0, tmp2, 0, count); + tmp = tmp2; + } + tmp[count] = line.substring(start, end); + count++; + start = end + 1; + if (count == limit) + break; + end = line.indexOf(separator, start); + } + if (count == tmp.length) { // check if array needs resize + final var tmp2 = new String[tmp.length + 1]; + System.arraycopy(tmp, 0, tmp2, 0, count); + tmp = tmp2; + } + tmp[count] = line.substring(start); + count++; + + if (count == tmp.length) { + return tmp; + } + final var result = new String[count]; + System.arraycopy(tmp, 0, result, 0, count); + return result; + } + + /** + * Very fast String splitting. + */ + public static List splitToList(final String line, final char separator) { + if (line.isEmpty()) + return LIST_WITH_EMPTY_STRING; + + final var result = new ArrayList(8); + int start = 0; + int end = line.indexOf(separator, 0); + while (end >= 0) { + result.add(line.substring(start, end)); + start = end + 1; + end = line.indexOf(separator, start); + } + result.add(line.substring(start)); + return result; + } + + public static int strcmp(final String a, final String b) { + final int result = a.compareTo(b); + if (result < 0) { + return -1; + } else if (result > 0) { + return 1; + } + return 0; + } + + public static int strArrCmp(@Nullable final List a, @Nullable final List b) { + if (a == null && b == null) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + final int len1 = a.size(); + final int len2 = b.size(); + if (len1 == len2) { + for (int i = 0; i < len1; i++) { + final int res = strcmp(a.get(i), b.get(i)); + if (res != 0) { + return res; + } + } + return 0; + } + return len1 - len2; + } + + public static String substringBefore(final String searchIn, final char searchFor, final String defaultValue) { + final int foundAt = searchIn.indexOf(searchFor); + return foundAt == -1 ? defaultValue : searchIn.substring(0, foundAt); + } + + /** + * @return "{SimpleClassName}{...fields...}" + */ + public static String toString(@Nullable final Object object, final Consumer fieldsBuilder) { + if (object == null) + return "null"; + final var sb = new StringBuilder(object.getClass().getSimpleName()); + sb.append('{'); + fieldsBuilder.accept(sb); + sb.append('}'); + return sb.toString(); + } + + private StringUtils() { + } } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IGrammarSource.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IGrammarSource.java index 75c8513a6..e9b12872e 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IGrammarSource.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IGrammarSource.java @@ -1,16 +1,14 @@ /** * Copyright (c) 2022 Sebastian Thomschke and others. - *

+ * * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - *

+ * * SPDX-License-Identifier: EPL-2.0 */ package org.eclipse.tm4e.core.registry; -import org.eclipse.jdt.annotation.Nullable; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -21,34 +19,38 @@ import java.io.StringReader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jdt.annotation.Nullable; public interface IGrammarSource { - /** - * Supported grammar content types - */ - enum ContentType { - JSON, - YAML, - XML - } + /** + * Supported grammar content types + */ + enum ContentType { + JSON, + YAML, + XML + } - private static ContentType guessFileFormat(final String fileName) { - final String extension = fileName.substring(fileName.lastIndexOf('.') + 1).trim().toLowerCase(); + private static ContentType guessFileFormat(final String fileName) { + final String extension = fileName.substring(fileName.lastIndexOf('.') + 1).trim().toLowerCase(); - return switch (extension) { - case "json" -> ContentType.JSON; - case "yaml", "yaml-tmlanguage", "yml" -> ContentType.YAML; - case "plist", "tmlanguage", "xml" -> ContentType.XML; - default -> throw new IllegalArgumentException("Unsupported file type: " + fileName); - }; - } + return switch (extension) { + case "json" -> ContentType.JSON; + case "yaml", "yaml-tmlanguage", "yml" -> ContentType.YAML; + case "plist", "tmlanguage", "xml" -> ContentType.XML; + default -> throw new IllegalArgumentException("Unsupported file type: " + fileName); + }; + } - static IGrammarSource fromFile(final File file) { - return fromFile(file, null, null); - } + static IGrammarSource fromFile(final File file) { + return fromFile(file, null, null); + } - static IGrammarSource fromFile(final File file, @Nullable final ContentType contentType, @Nullable final Charset charset) { + static IGrammarSource fromFile(final File file, @Nullable final ContentType contentType, @Nullable final Charset charset) { final var filePath = file.getAbsolutePath(); final var contentType1 = contentType == null ? guessFileFormat(filePath) : contentType; @@ -59,17 +61,17 @@ public Reader getReader() throws IOException { return new BufferedReader(new InputStreamReader(new FileInputStream(file), charset == null ? StandardCharsets.UTF_8 : charset)); } - @Override - public String getFilePath() { - return filePath; - } + @Override + public String getFilePath() { + return filePath; + } - @Override - public ContentType getContentType() { - return contentType1; - } - }; - } + @Override + public ContentType getContentType() { + return contentType1; + } + }; + } static IGrammarSource fromInputStream(InputStream stream, @Nullable final String fileName, @Nullable final Charset charset) { @@ -117,58 +119,58 @@ static IGrammarSource fromResource(final Class clazz, final String resourceNa return fromResource(clazz, resourceName, null, null); } - /** - * @throws IllegalArgumentException if the content type is unsupported or cannot be determined - */ - static IGrammarSource fromResource(final Class clazz, final String resourceName, @Nullable final ContentType contentType, - @Nullable final Charset charset) { - - final var contentType1 = contentType == null ? guessFileFormat(resourceName) : contentType; - return new IGrammarSource() { - @Override - public Reader getReader() { - return new BufferedReader(new InputStreamReader( - clazz.getResourceAsStream(resourceName), - charset == null ? StandardCharsets.UTF_8 : charset)); - } - - @Override - public String getFilePath() { - return resourceName; - } - - @Override - public ContentType getContentType() { - return contentType1; - } - }; - } - - static IGrammarSource fromString(final ContentType contentType, final String content) { - - return new IGrammarSource() { - @Override - public Reader getReader() { - return new StringReader(content); - } - - @Override - public String getFilePath() { - return "string." + contentType.name().toLowerCase(); - } - - @Override - public ContentType getContentType() { - return contentType; - } - }; - } - - default ContentType getContentType() { - return guessFileFormat(getFilePath()); - } - - String getFilePath(); - - Reader getReader() throws IOException; + /** + * @throws IllegalArgumentException if the content type is unsupported or cannot be determined + */ + static IGrammarSource fromResource(final Class clazz, final String resourceName, @Nullable final ContentType contentType, + @Nullable final Charset charset) { + + final var contentType1 = contentType == null ? guessFileFormat(resourceName) : contentType; + return new IGrammarSource() { + @Override + public Reader getReader() { + return new BufferedReader(new InputStreamReader( + clazz.getResourceAsStream(resourceName), + charset == null ? StandardCharsets.UTF_8 : charset)); + } + + @Override + public String getFilePath() { + return resourceName; + } + + @Override + public ContentType getContentType() { + return contentType1; + } + }; + } + + static IGrammarSource fromString(final ContentType contentType, final String content) { + + return new IGrammarSource() { + @Override + public Reader getReader() { + return new StringReader(content); + } + + @Override + public String getFilePath() { + return "string." + contentType.name().toLowerCase(); + } + + @Override + public ContentType getContentType() { + return contentType; + } + }; + } + + default ContentType getContentType() { + return guessFileFormat(getFilePath()); + } + + String getFilePath(); + + Reader getReader() throws IOException; } diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IThemeSource.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IThemeSource.java index 3b1311546..5781aefd2 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IThemeSource.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/IThemeSource.java @@ -19,7 +19,8 @@ import java.io.StringReader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; - +import java.nio.file.Files; +import java.nio.file.Path; import org.eclipse.jdt.annotation.Nullable; diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/Registry.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/Registry.java index 6721bda71..5ebfde243 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/Registry.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/registry/Registry.java @@ -23,13 +23,14 @@ import org.eclipse.tm4e.core.TMException; import org.eclipse.tm4e.core.grammar.IGrammar; import org.eclipse.tm4e.core.internal.grammar.BalancedBracketSelectors; -import org.eclipse.tm4e.core.internal.grammar.dependencies.AbsoluteRuleReference; import org.eclipse.tm4e.core.internal.grammar.dependencies.ScopeDependencyProcessor; import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammar; import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader; import org.eclipse.tm4e.core.internal.registry.SyncRegistry; import org.eclipse.tm4e.core.internal.theme.Theme; import org.eclipse.tm4e.core.internal.theme.raw.RawThemeReader; +import org.eclipse.tm4e.core.internal.utils.ScopeNames; + import java.util.HashMap; import java.util.List; @@ -41,8 +42,9 @@ * The registry that will hold all grammars. * * @see - * github.com/microsoft/vscode-textmate/blob/main/src/main.ts + * "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/main.ts#L54"> + * github.com/microsoft/vscode-textmate/blob/main/src/main.ts + * */ public final class Registry { @@ -146,11 +148,13 @@ private IGrammar _loadGrammar( @Nullable final Map embeddedLanguages, @Nullable final Map tokenTypes, @Nullable final BalancedBracketSelectors balancedBracketSelectors) { + + if (!_loadSingleGrammar(initialScopeName)) + return null; + final var dependencyProcessor = new ScopeDependencyProcessor(this._syncRegistry, initialScopeName); while (!dependencyProcessor.Q.isEmpty()) { - for (AbsoluteRuleReference request : dependencyProcessor.Q) { - this._loadSingleGrammar(request.scopeName); - } + dependencyProcessor.Q.forEach(request -> this._loadSingleGrammar(request.scopeName)); dependencyProcessor.processQueue(); } @@ -162,12 +166,17 @@ private IGrammar _loadGrammar( balancedBracketSelectors); } - private void _loadSingleGrammar(final String scopeName) { - this._ensureGrammarCache.computeIfAbsent(scopeName, this::_doLoadSingleGrammar); + private boolean _loadSingleGrammar(final String scopeName) { + return this._ensureGrammarCache.computeIfAbsent(scopeName, this::_doLoadSingleGrammar); } private boolean _doLoadSingleGrammar(final String scopeName) { - final var grammarSource = this._options.getGrammarSource(scopeName); + var grammarSource = this._options.getGrammarSource(scopeName); + if (grammarSource == null) { + final var scopeNameWithoutContributor = ScopeNames.withoutContributor(scopeName); + if (!scopeNameWithoutContributor.equals(scopeName)) + grammarSource = this._options.getGrammarSource(scopeNameWithoutContributor); + } if (grammarSource == null) { LOGGER.i("No grammar source for scope [{0}]", scopeName); return false; @@ -203,8 +212,7 @@ public IGrammar addGrammar( injections == null || injections.isEmpty() ? this._options.getInjections(rawGrammar.getScopeName()) : injections); - return castNonNull( - this._grammarForScopeName(rawGrammar.getScopeName(), initialLanguage, embeddedLanguages, null, null)); + return castNonNull(this._grammarForScopeName(rawGrammar.getScopeName(), initialLanguage, embeddedLanguages, null, null)); } catch (final Exception ex) { throw new TMException("Loading grammar from [" + source.getFilePath() + "] failed: " + ex.getMessage(), ex); diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPair.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPair.java new file mode 100644 index 000000000..c95eb6d59 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPair.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2022 Sebastian Thomschke and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Initial code from https://github.com/microsoft/vscode-textmate/ + * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. + * Initial license: MIT + * + * Contributors: + * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license + * - Sebastian Thomschke - translation and adaptation to Java + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +/** + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L196 + */ +public class AutoClosingPair extends CharacterPair { + + public AutoClosingPair(final String open, final String close) { + super(open, close); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java new file mode 100644 index 000000000..1ccef74ff --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import java.util.List; + +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L201 + */ +public final class AutoClosingPairConditional extends AutoClosingPair { + + public final List notIn; + + public AutoClosingPairConditional(final String open, final String close, final List notIn) { + super(open, close); + this.notIn = notIn; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("open=").append(open).append(", ") + .append("close=").append(close).append(", ") + .append("notIn=").append(notIn)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java new file mode 100644 index 000000000..8c4efe8c3 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * A tuple of two characters, like a pair of opening and closing brackets. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L194 + */ +public class CharacterPair { + + public final String open; + public final String close; + + public CharacterPair(final String opening, final String closing) { + this.open = opening; + this.close = closing; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("open=").append(open).append(", ") + .append("close=").append(close)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CommentRule.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CommentRule.java new file mode 100644 index 000000000..f5d6624de --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CommentRule.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2018 Red Hat Inc. and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Lucas Bullen (Red Hat Inc.) - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * Describes how comments for a language work. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L13 + */ +public final class CommentRule { + + /** + * The line comment token, like `// this is a comment` + */ + @Nullable + public final String lineComment; + + /** + * The block comment character pair, like `/* block comment */` + */ + @Nullable + public final CharacterPair blockComment; + + public CommentRule(@Nullable final String lineComment, @Nullable final CharacterPair blockComment) { + this.lineComment = lineComment; + this.blockComment = blockComment; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("lineComment=").append(lineComment).append(", ") + .append("blockComment=").append(blockComment)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CompleteEnterAction.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CompleteEnterAction.java new file mode 100644 index 000000000..d09cb0a79 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CompleteEnterAction.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L250 + */ +public final class CompleteEnterAction extends EnterAction { + + /** + * The line's indentation minus removeText + */ + public final String indentation; + + public CompleteEnterAction(IndentAction indentAction, @Nullable String appendText, @Nullable Integer removeText, + final String indentation) { + super(indentAction, appendText, removeText); + this.indentation = indentation; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("indentAction=").append(indentAction).append(", ") + .append("appendText=").append(appendText).append(", ") + .append("removeText=").append(removeText).append(", ") + .append("indentation=").append(indentation)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/EnterAction.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/EnterAction.java new file mode 100644 index 000000000..c97d50565 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/EnterAction.java @@ -0,0 +1,100 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * Describes what to do when pressing Enter. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L232 + */ +public class EnterAction { + + public enum IndentAction { + /** + * Insert new line and copy the previous line's indentation. + */ + None, + /** + * Insert new line and indent once (relative to the previous line's + * indentation). + */ + Indent, + /** + * Insert two new lines: - the first one indented which will hold the cursor - + * the second one at the same indentation level + */ + IndentOutdent, + /** + * Insert new line and outdent once (relative to the previous line's + * indentation). + */ + Outdent; + + public static IndentAction get(final @Nullable String value) { + // see + // https://github.com/microsoft/vscode/blob/13ba7bb446a638d37ebccb1a7d74e31c32bb9790/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts#L341 + if (value == null) { + return IndentAction.None; + } + switch (value) { + case "none": + return IndentAction.None; + case "indent": + return IndentAction.Indent; + case "indentOutdent": + return IndentAction.IndentOutdent; + case "outdent": + return IndentAction.Outdent; + default: + return IndentAction.None; + } + } + } + + /** + * Describe what to do with the indentation. + */ + public final IndentAction indentAction; + + /** + * Describes text to be appended after the new line and after the indentation. + */ + public @Nullable String appendText; + + /** + * Describes the number of characters to remove from the new line's indentation. + */ + public final @Nullable Integer removeText; + + public EnterAction(final IndentAction indentAction) { + this(indentAction, null, null); + } + + public EnterAction(final IndentAction indentAction, final @Nullable String appendText, final @Nullable Integer removeText) { + this.indentAction = indentAction; + this.appendText = appendText; + this.removeText = removeText; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("indentAction=").append(indentAction).append(", ") + .append("appendText=").append(appendText).append(", ") + .append("removeText=").append(removeText)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/FoldingRules.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/FoldingRules.java new file mode 100644 index 000000000..415bfca5f --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/FoldingRules.java @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2018 Red Hat Inc. and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Lucas Bullen (Red Hat Inc.) - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * Describes folding rules for a language. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L139 + */ +public final class FoldingRules { + + /** + * Used by the indentation based strategy to decide whether empty lines belong to the previous or the next block. + * A language adheres to the off-side rule if blocks in that language are expressed by their indentation. + * See [wikipedia](https://en.wikipedia.org/wiki/Off-side_rule) for more information. + */ + public final boolean offSide; + public final RegExPattern markersStart; + public final RegExPattern markersEnd; + + public FoldingRules(final boolean offSide, final RegExPattern markersStart, final RegExPattern markersEnd) { + this.offSide = offSide; + this.markersStart = markersStart; + this.markersEnd = markersEnd; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("markersStart=").append(markersStart).append(", ") + .append("markersEnd=").append(markersEnd).append(", ") + .append("offSide=").append(offSide)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentForEnter.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentForEnter.java new file mode 100644 index 000000000..9763a5f62 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentForEnter.java @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2024 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Initial code from https://github.com/microsoft/vscode/ + * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. + * Initial license: MIT + * + * Contributors: + * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license + * - Sebastian Thomschke - translation and adaptation to Java + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * Describes indentation rules for a language. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/autoIndent.ts#L278 + */ +public class IndentForEnter { + + public final String beforeEnter; + public final String afterEnter; + + public IndentForEnter(final String beforeEnter, final String afterEnter) { + this.beforeEnter = beforeEnter; + this.afterEnter = afterEnter; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("beforeEnter=").append(beforeEnter).append(", ") + .append("afterEnter=").append(afterEnter)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentationRules.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentationRules.java new file mode 100644 index 000000000..5f98d89d5 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/IndentationRules.java @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2024 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Initial code from https://github.com/microsoft/vscode/ + * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. + * Initial license: MIT + * + * Contributors: + * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license + * - Sebastian Thomschke - translation and adaptation to Java + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.jdt.annotation.Nullable; + +/** + * Describes indentation rules for a language. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L105 + */ +public class IndentationRules { + + /** + * If a line matches this pattern, then all the lines after it should be unindented once (until another rule matches). + */ + public final RegExPattern decreaseIndentPattern; + + /** + * If a line matches this pattern, then all the lines after it should be indented once (until another rule matches). + */ + public final RegExPattern increaseIndentPattern; + + /** + * If a line matches this pattern, then **only the next line** after it should be indented once. + */ + public final @Nullable RegExPattern indentNextLinePattern; + + /** + * If a line matches this pattern, then its indentation should not be changed and it should not be evaluated against the other rules. + */ + public final @Nullable RegExPattern unIndentedLinePattern; + + public IndentationRules(final RegExPattern decreaseIndentPattern, final RegExPattern increaseIndentPattern, + final @Nullable RegExPattern indentNextLinePattern, final @Nullable RegExPattern unIndentedLinePattern) { + this.decreaseIndentPattern = decreaseIndentPattern; + this.increaseIndentPattern = increaseIndentPattern; + this.indentNextLinePattern = indentNextLinePattern; + this.unIndentedLinePattern = unIndentedLinePattern; + } +} \ No newline at end of file diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java new file mode 100644 index 000000000..d4694aa3e --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java @@ -0,0 +1,457 @@ +/** + * Copyright (c) 2018 Red Hat Inc. and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Lucas Bullen (Red Hat Inc.) - initial API and implementation + * Sebastian Thomschke (Vegard IT) - improve JSON parsing tolerance, add indentation rules parsing + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.*; + +import java.io.BufferedReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.languageconfiguration.internal.model.EnterAction.IndentAction; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import io.github.rosemoe.sora.util.Logger; + +/** + * The language configuration interface defines the contract between extensions and various editor features, like + * automatic bracket insertion, automatic indentation etc. + * + * @see code.visualstudio.com/api/language-extensions/language-configuration-guide + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts + */ +public class LanguageConfiguration { + + private static final Logger log = Logger.instance(LanguageConfiguration.class.getName()); + + private static String removeTrailingCommas(String jsonString) { + /* matches: + * -------------- + * }, + * } + * -------------- + * as well as: + * -------------- + * }, + * // foo + * // bar + * } + * -------------- + */ + return jsonString.replaceAll("(,)(\\s*\\n(\\s*\\/\\/.*\\n)*\\s*[\\]}])", "$2"); + } + + /** + * See JSON format at https://code.visualstudio.com/api/language-extensions/language-configuration-guide + * + * @return an instance of {@link LanguageConfiguration} loaded from the VSCode language-configuration.json file + * reader. + */ + @NonNullByDefault({}) + public static @Nullable LanguageConfiguration load(@NonNull final Reader reader) { + // GSON does not support trailing commas so we have to manually remove them -> maybe better switch to jackson json parser? + final var jsonString = removeTrailingCommas(new BufferedReader(reader).lines().collect(Collectors.joining("\n"))); + final LanguageConfiguration langCfg = new GsonBuilder() + .registerTypeAdapter(String.class, (JsonDeserializer) (json, typeOfT, context) -> { + if (json.isJsonObject()) { + /* for example: + * "wordPattern": { + * "pattern": "...", + * "flags": "..." + * }, + */ + final var jsonObj = json.getAsJsonObject(); + return jsonObj.has("pattern") && jsonObj.get("pattern").isJsonPrimitive() // + ? jsonObj.get("pattern").getAsString() + : null; + } + + /* for example: + * "wordPattern": "...", + */ + return json.getAsString(); + }) + + .registerTypeAdapter(OnEnterRule.class, (JsonDeserializer) (json, typeOfT, context) -> { + if (!json.isJsonObject()) { + return null; + } + + final var jsonObj = json.getAsJsonObject(); + final var beforeText = getAsPattern(jsonObj.get("beforeText")); //$NON-NLS-1$ + if (beforeText == null) { + return null; + } + + final var actionElem = jsonObj.get("action"); //$NON-NLS-1$ + if (actionElem != null && actionElem.isJsonObject()) { + final var actionJsonObj = actionElem.getAsJsonObject(); + final var indentActionString = getAsString(actionJsonObj.get("indent")); //$NON-NLS-1$ + if (indentActionString != null) { + final var afterText = getAsPattern(jsonObj.get("afterText")); //$NON-NLS-1$ + final var previousLineText = getAsPattern(jsonObj.get("previousLineText")); //$NON-NLS-1$ + final var indentAction = IndentAction.get(indentActionString); + final var appendText = getAsString(actionJsonObj.get("appendText")); //$NON-NLS-1$ + final var removeText = getAsInteger(actionJsonObj.get("removeText")); //$NON-NLS-1$ + final var action = new EnterAction(indentAction, appendText, removeText); + return new OnEnterRule(beforeText, afterText, previousLineText, action); + } + } + return null; + }) + + .registerTypeAdapter(CommentRule.class, (JsonDeserializer) (json, typeOfT, context) -> { + if (!json.isJsonObject()) { + return null; + } + + // ex: {"lineComment": "//","blockComment": [ "/*", "*/" ]} + final var jsonObj = json.getAsJsonObject(); + final var lineComment = getAsString(jsonObj.get("lineComment")); //$NON-NLS-1$ + final var blockCommentElem = jsonObj.get("blockComment"); //$NON-NLS-1$ + CharacterPair blockComment = null; + if (blockCommentElem != null && blockCommentElem.isJsonArray()) { + final var blockCommentArray = blockCommentElem.getAsJsonArray(); + if (blockCommentArray.size() == 2) { + final var blockCommentStart = getAsString(blockCommentArray.get(0)); + final var blockCommentEnd = getAsString(blockCommentArray.get(1)); + if (blockCommentStart != null && blockCommentEnd != null) { + blockComment = new CharacterPair(blockCommentStart, blockCommentEnd); + } + } + } + + return lineComment == null && blockComment == null + ? null + : new CommentRule(lineComment, blockComment); + }) + + .registerTypeAdapter(CharacterPair.class, (JsonDeserializer) (json, typeOfT, + context) -> { + if (!json.isJsonArray()) { + return null; + } + + // ex: ["{","}"] + final var charsPair = json.getAsJsonArray(); + if (charsPair.size() != 2) { + return null; + } + + final var open = getAsString(charsPair.get(0)); + final var close = getAsString(charsPair.get(1)); + + return open == null || close == null + ? null + : new CharacterPair(open, close); + }) + + .registerTypeAdapter(AutoClosingPair.class, (JsonDeserializer) (json, typeOfT, + context) -> { + String open = null; + String close = null; + if (json.isJsonArray()) { + // ex: ["{","}"] + final var charsPair = json.getAsJsonArray(); + if (charsPair.size() != 2) { + return null; + } + open = getAsString(charsPair.get(0)); + close = getAsString(charsPair.get(1)); + } else if (json.isJsonObject()) { + // ex: {"open":"'","close":"'"} + final var autoClosePair = json.getAsJsonObject(); + open = getAsString(autoClosePair.get("open")); //$NON-NLS-1$ + close = getAsString(autoClosePair.get("close")); //$NON-NLS-1$ + } + + return open == null || close == null + ? null + : new AutoClosingPair(open, close); + }) + + .registerTypeAdapter(AutoClosingPairConditional.class, (JsonDeserializer) ( + json, typeOfT, context) -> { + final var notInList = new ArrayList(2); + String open = null; + String close = null; + if (json.isJsonArray()) { + // ex: ["{","}"] + final var charsPair = json.getAsJsonArray(); + if (charsPair.size() != 2) { + return null; + } + open = getAsString(charsPair.get(0)); + close = getAsString(charsPair.get(1)); + } else if (json.isJsonObject()) { + // ex: {"open":"'","close":"'", "notIn": ["string", "comment"]} + final var autoClosePair = json.getAsJsonObject(); + open = getAsString(autoClosePair.get("open")); //$NON-NLS-1$ + close = getAsString(autoClosePair.get("close")); //$NON-NLS-1$ + final var notInElem = autoClosePair.get("notIn"); //$NON-NLS-1$ + if (notInElem != null && notInElem.isJsonArray()) { + for (final JsonElement elem : notInElem.getAsJsonArray()) { + final var string = getAsString(elem); + if (string != null) { + notInList.add(string); + } + } + } + } + + return open == null || close == null + ? null + : new AutoClosingPairConditional(open, close, notInList); + }) + + .registerTypeAdapter(FoldingRules.class, (JsonDeserializer) (json, typeOfT, context) -> { + if (!json.isJsonObject()) { + return null; + } + + // ex: {"offSide": true, "markers": {"start": "^\\s*/", "end": "^\\s*"}} + final var jsonObj = json.getAsJsonObject(); + final var markersElem = jsonObj.get("markers"); //$NON-NLS-1$ + if (markersElem != null && markersElem.isJsonObject()) { + final var markersObj = markersElem.getAsJsonObject(); + final var startMarker = getAsPattern(markersObj.get("start")); //$NON-NLS-1$ + final var endMarker = getAsPattern(markersObj.get("end")); //$NON-NLS-1$ + if (startMarker != null && endMarker != null) { + final var offSide = getAsBoolean(jsonObj.get("offSide"), false); //$NON-NLS-1$ + return new FoldingRules(offSide, startMarker, endMarker); + } + } + return null; + }) + + .registerTypeAdapter(IndentationRules.class, (JsonDeserializer) (json, typeT, context) -> { + if (!json.isJsonObject()) { + return null; + } + + final var jsonObj = json.getAsJsonObject(); + final var decreaseIndentPattern = getAsPattern(jsonObj.get("decreaseIndentPattern")); + if (decreaseIndentPattern == null) + return null; + final var increaseIndentPattern = getAsPattern(jsonObj.get("increaseIndentPattern")); + if (increaseIndentPattern == null) + return null; + + return new IndentationRules( + decreaseIndentPattern, + increaseIndentPattern, + getAsPattern(jsonObj.get("indentNextLinePattern")), + getAsPattern(jsonObj.get("unIndentedLinePattern"))); + }) + .create() + .fromJson(jsonString, LanguageConfiguration.class); + + if (castNullable(langCfg.autoClosingPairs) == null) { + langCfg.autoClosingPairs = Collections.emptyList(); + } else { + langCfg.autoClosingPairs.removeIf(Objects::isNull); + } + + if (castNullable(langCfg.brackets) == null) { + langCfg.brackets = Collections.emptyList(); + } else { + langCfg.brackets.removeIf(Objects::isNull); + } + + if (castNullable(langCfg.onEnterRules) == null) { + langCfg.onEnterRules = Collections.emptyList(); + } else { + langCfg.onEnterRules.removeIf(Objects::isNull); + } + + if (castNullable(langCfg.surroundingPairs) == null) { + langCfg.surroundingPairs = Collections.emptyList(); + } else { + langCfg.surroundingPairs.removeIf(Objects::isNull); + } + + if (castNullable(langCfg.colorizedBracketPairs) == null) { + langCfg.colorizedBracketPairs = Collections.emptyList(); + } else { + langCfg.colorizedBracketPairs.removeIf(Objects::isNull); + } + return langCfg; + } + + private static @Nullable RegExPattern getAsPattern(@Nullable final JsonElement element) { + if (element == null) { + return null; + } + if (element.isJsonObject()) { + // ex : { "pattern": "^<\\/([_:\\w][_:\\w-.\\d]*)\\s*>", "flags": "i" } + final var pattern = getAsString(((JsonObject) element).get("pattern")); + if (pattern == null) { + return null; + } + final var flags = getAsString(((JsonObject) element).get("flags")); + //return flags != null ? pattern + "(?" + flags + ")" : pattern; + return RegExPattern.of(pattern, flags); + } + // ex : "^<\\/([_:\\w][_:\\w-.\\d]*)\\s*>" + return RegExPattern.ofNullable(getAsString(element), null); + } + + private static @Nullable String getAsString(@Nullable final JsonElement element) { + if (element != null) + try { + return element.getAsString(); + } catch (final Exception ex) { + log.e("Failed to convert JSON element [" + element + "] to String.", ex); + } + return null; + } + + private static boolean getAsBoolean(@Nullable final JsonElement element, final boolean defaultValue) { + if (element != null) + try { + return element.getAsBoolean(); + } catch (final Exception ex) { + log.e("Failed to convert JSON element [" + element + "] to boolean.", ex); + } + return defaultValue; + } + + private static @Nullable Integer getAsInteger(@Nullable final JsonElement element) { + if (element != null) + try { + return element.getAsInt(); + } catch (final Exception ex) { + log.e("Failed to convert JSON element [" + element + "] to Integer.", ex); + } + return null; + } + + private @Nullable CommentRule comments; + + /** + * Returns the language's comments. The comments are used by {@link AutoClosingPairConditional} when + * notIn contains comment + * + * @return the language's comments. + */ + public @Nullable CommentRule getComments() { + return comments; + } + + private List brackets = lazyNonNull(); + + /** + * Returns the language's brackets. This configuration implicitly affects pressing Enter around these brackets. + * + * @return the language's brackets + */ + public List getBrackets() { + return brackets; + } + + private @Nullable String wordPattern; + + /** + * Returns the language's definition of a word. This is the regex used when referring to a word. + * + * @return the language's word pattern. + */ + public @Nullable String getWordPattern() { + return wordPattern; + } + + private @Nullable IndentationRules indentationRules; + + /** + * The language's indentation settings. + */ + public @Nullable IndentationRules getIndentationRules() { + return indentationRules; + } + + private List onEnterRules = lazyNonNull(); + + /** + * Returns the language's rules to be evaluated when pressing Enter. + * + * @return the language's rules to be evaluated when pressing Enter. + */ + public List getOnEnterRules() { + return onEnterRules; + } + + private List autoClosingPairs = lazyNonNull(); + + /** + * Returns the language's auto closing pairs. The 'close' character is automatically inserted with the 'open' + * character is typed. If not set, the configured brackets will be used. + * + * @return the language's auto closing pairs. + */ + public List getAutoClosingPairs() { + return autoClosingPairs; + } + + private List surroundingPairs = lazyNonNull(); + + /** + * Returns the language's surrounding pairs. When the 'open' character is typed on a selection, the selected string + * is surrounded by the open and close characters. If not set, the autoclosing pairs settings will be used. + * + * @return the language's surrounding pairs. + */ + public List getSurroundingPairs() { + return surroundingPairs; + } + + private List colorizedBracketPairs = lazyNonNull(); + + /** + * Defines a list of bracket pairs that are colorized depending on their nesting level. + * If not set, the configured brackets will be used. + */ + public List getColorizedBracketPairs() { + return colorizedBracketPairs; + } + + private @Nullable String autoCloseBefore; + + public @Nullable String getAutoCloseBefore() { + return autoCloseBefore; + } + + private @Nullable FoldingRules folding; + + /** + * Returns the language's folding rules. + * + * @return the language's folding rules. + */ + public @Nullable FoldingRules getFolding() { + return folding; + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/OnEnterRule.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/OnEnterRule.java new file mode 100644 index 000000000..b2586a5c1 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/OnEnterRule.java @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + * Sebastian Thomschke (Vegard IT GmbH) - add previousLineText support + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.TMException; +import org.eclipse.tm4e.core.internal.utils.StringUtils; + +/** + * Describes a rule to be evaluated when pressing Enter. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L157 + */ +public final class OnEnterRule { + + /** + * This rule will only execute if the text before the cursor matches this regular expression. + */ + public final RegExPattern beforeText; + + /** + * This rule will only execute if the text after the cursor matches this regular expression. + */ + public final @Nullable RegExPattern afterText; + + /** + * This rule will only execute if the text above the current line matches this regular expression. + */ + + public final @Nullable RegExPattern previousLineText; + + /** + * The action to execute. + */ + public final EnterAction action; + + public OnEnterRule(final RegExPattern beforeText, final @Nullable RegExPattern afterText, final @Nullable RegExPattern previousLineText, + final EnterAction action) { + this.beforeText = beforeText; + this.afterText = afterText; + this.previousLineText = previousLineText; + this.action = action; + } + + /** + * Only for unit tests + * + * @throws TMException if beforeText, afterText or previousLineText contain invalid regex pattern + */ + OnEnterRule(final String beforeText, final @Nullable String afterText, final @Nullable String previousLineText, + final EnterAction action) { + this.beforeText = RegExPattern.of(beforeText); + this.afterText = afterText == null ? null : RegExPattern.of(afterText); + this.previousLineText = previousLineText == null ? null : RegExPattern.of(previousLineText); + this.action = action; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> sb + .append("beforeText=").append(beforeText).append(", ") + .append("afterText=").append(afterText).append(", ") + .append("previousLineText=").append(previousLineText).append(", ") + .append("action=").append(action)); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/RegExPattern.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/RegExPattern.java new file mode 100644 index 000000000..610d42248 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/RegExPattern.java @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2024 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * - Sebastian Thomschke - initial implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.model; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.TMException; +import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp; +import org.eclipse.tm4e.core.internal.oniguruma.OnigString; + +public abstract class RegExPattern { + + private static final class JavaRegExPattern extends RegExPattern { + final Pattern pattern; + + JavaRegExPattern(final String pattern, final @Nullable String flags) throws PatternSyntaxException { + // maybe unsupported in android? + this.pattern = Pattern.compile(flags == null ? pattern : pattern + "(?" + flags + ")"); + } + + @Override + public boolean matchesFully(final String text) { + return pattern.matcher(text).matches(); + } + + @Override + public boolean matchesPartially(final String text) { + return pattern.matcher(text).find(); + } + + @Override + public String pattern() { + return pattern.pattern(); + } + } + + private static final class OnigRegExPattern extends RegExPattern { + final OnigRegExp regex; + + OnigRegExPattern(final String pattern, final @Nullable String flags) throws PatternSyntaxException { + this.regex = new OnigRegExp(pattern, flags != null && flags.contains("i")); + } + + @Override + public boolean matchesFully(final String text) { + final var result = regex.search(OnigString.of(text), 0); + return result != null && result.count() == 1 && result.lengthAt(0) == text.length(); + } + + @Override + public boolean matchesPartially(final String text) { + return regex.search(OnigString.of(text), 0) != null; + } + + @Override + public String pattern() { + return regex.pattern(); + } + } + + /** + * @param pattern {@link Pattern} or {@link OnigRegExp} compatible pattern + * + * @throws TMException if pattern parsing fails + */ + public static RegExPattern of(final String pattern) { + return of(pattern, null); + } + + /** + * @param pattern {@link Pattern} or {@link OnigRegExp} compatible pattern + * + * @throws TMException if pattern parsing fails + */ + public static RegExPattern of(final String pattern, final @Nullable String flags) { + try { + return new JavaRegExPattern(pattern, flags); + } catch (Exception ex) { + // try onigurama as fallback + return new OnigRegExPattern(pattern, flags); + } + } + + /** + * @param pattern {@link Pattern} or {@link OnigRegExp} compatible pattern + * + * @return null if pattern is null or the pattern is invalid + */ + public static @Nullable RegExPattern ofNullable(final @Nullable String pattern) { + return ofNullable(pattern, null); + } + + /** + * @param pattern {@link Pattern} or {@link OnigRegExp} compatible pattern + * + * @return null if pattern is null or the pattern is invalid + */ + public static @Nullable RegExPattern ofNullable(final @Nullable String pattern, final @Nullable String flags) { + if (pattern != null) { + try { + return new JavaRegExPattern(pattern, flags); + } catch (Exception ex) { + try { + // try onigurama as fallback + return new OnigRegExPattern(pattern, flags); + } catch (Exception ex1) { + ex1.printStackTrace(); + } + } + } + return null; + } + + public abstract boolean matchesFully(String text); + + public abstract boolean matchesPartially(String text); + + public abstract String pattern(); + + @Override + public String toString() { + return pattern(); + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CharacterPairSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CharacterPairSupport.java new file mode 100644 index 000000000..4dad428db --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CharacterPairSupport.java @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Initial code from https://github.com/microsoft/vscode/ + * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. + * Initial license: MIT + * + * Contributors: + * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license + * - Angelo Zerr - translation and adaptation to Java + */ +package org.eclipse.tm4e.languageconfiguration.internal.supports; + +import java.util.Collections; +import java.util.List; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.languageconfiguration.internal.model.AutoClosingPair; +import org.eclipse.tm4e.languageconfiguration.internal.model.AutoClosingPairConditional; +import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration; + +/** + * The "character pair" support. + * + * @see + * https://github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/characterPair.ts + */ +public final class CharacterPairSupport { + + public static final String DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED = ";:.,=}])> \r\n\t"; + public static final String DEFAULT_AUTOCLOSE_BEFORE_WHITESPACE = " \r\n\t"; + + public final List autoClosingPairs; + public final List surroundingPairs; + public final String autoCloseBefore; + + @SuppressWarnings("unchecked") + public CharacterPairSupport(final LanguageConfiguration config) { + final var autoClosingPairs = config.getAutoClosingPairs(); + if (!autoClosingPairs.isEmpty()) { + this.autoClosingPairs = autoClosingPairs; + } else { + final var brackets = config.getBrackets(); + if (!brackets.isEmpty()) { + this.autoClosingPairs = brackets.stream() + .map(el -> new AutoClosingPairConditional(el.open, el.close, Collections.emptyList())) + .toList(); + } else { + this.autoClosingPairs = Collections.emptyList(); + } + } + + final var autoCloseBefore = config.getAutoCloseBefore(); + this.autoCloseBefore = autoCloseBefore != null + ? autoCloseBefore + : CharacterPairSupport.DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED; + + final var surroundingPairs = config.getSurroundingPairs(); + this.surroundingPairs = !surroundingPairs.isEmpty() + ? surroundingPairs + : (List) (List) this.autoClosingPairs; + } + + /** + * TODO not declared in upstream project + */ + @Nullable + public AutoClosingPairConditional getAutoClosingPair(final String text, final int offset, + final String newCharacter) { + if (newCharacter.isEmpty()) { + return null; + } + for (final AutoClosingPairConditional autoClosingPair : autoClosingPairs) { + final String opening = autoClosingPair.open; + if (!opening.endsWith(newCharacter)) { + continue; + } + if (opening.length() > 1) { + final String offsetPrefix = text.substring(0, offset); + if (!offsetPrefix.endsWith(opening.substring(0, opening.length() - 1))) { + continue; + } + } + return autoClosingPair; + } + return null; + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CommentSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CommentSupport.java new file mode 100644 index 000000000..438b551f3 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/CommentSupport.java @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2018 Red Hat Inc. and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + *

+ * SPDX-License-Identifier: EPL-2.0 + *

+ * Contributors: + * Lucas Bullen (Red Hat Inc.) - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.supports; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.languageconfiguration.internal.model.CharacterPair; +import org.eclipse.tm4e.languageconfiguration.internal.model.CommentRule; + +import io.github.rosemoe.sora.text.Content; + +public final class CommentSupport { + + @Nullable + private final CommentRule comments; + + public CommentSupport(@Nullable final CommentRule comments) { + this.comments = comments; + } + + private boolean isInComment(final Content text, final int offset) { + try { + if (isInBlockComment(text.subSequence(0, offset).toString())) { + return true; + } + var indexer = text.getIndexer(); + final int line = indexer.getCharLine(offset); + final int lineOffset = indexer.getCharIndex(line,0); + return isInLineComment(text.subSequence(lineOffset, offset - lineOffset).toString()); + } catch (final Exception e) { + return false; + } + } + + @Nullable + public String getLineComment() { + final var comments = this.comments; + return comments == null ? null : comments.lineComment; + } + + @Nullable + public CharacterPair getBlockComment() { + final var comments = this.comments; + return comments == null ? null : comments.blockComment; + } + + private boolean isInLineComment(final String indexLinePrefix) { + final var comments = this.comments; + if (comments == null) + return false; + return indexLinePrefix.indexOf(comments.lineComment) != -1; + } + + private boolean isInBlockComment(final String indexPrefix) { + final var comments = this.comments; + if (comments == null) + return false; + + final var blockComment = comments.blockComment; + if (blockComment == null) + return false; + + final String commentOpen = blockComment.open; + final String commentClose = blockComment.close; + int index = indexPrefix.indexOf(commentOpen); + while (index != -1 && index < indexPrefix.length()) { + final int closeIndex = indexPrefix.indexOf(commentClose, index + commentOpen.length()); + if (closeIndex == -1) { + return true; + } + index = indexPrefix.indexOf(commentOpen, closeIndex + commentClose.length()); + } + return false; + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/IndentRulesSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/IndentRulesSupport.java new file mode 100644 index 000000000..de7ad77c9 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/IndentRulesSupport.java @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2024 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Initial code from https://github.com/microsoft/vscode/ + * Initial copyright Copyright (C) Microsoft Corporation. All rights reserved. + * Initial license: MIT + * + * Contributors: + * - Microsoft Corporation: Initial code, written in TypeScript, licensed under MIT license + * - Sebastian Thomschke - translation and adaptation to Java + */ +package org.eclipse.tm4e.languageconfiguration.internal.supports; + +import org.eclipse.tm4e.languageconfiguration.internal.model.IndentationRules; + +/** + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/indentRules.ts + */ +public class IndentRulesSupport { + + public static final class IndentConsts { + public static final int INCREASE_MASK = 0b00000001; + public static final int DECREASE_MASK = 0b00000010; + public static final int INDENT_NEXTLINE_MASK = 0b00000100; + public static final int UNINDENT_MASK = 0b00001000; + } + + private final IndentationRules _indentationRules; + + public IndentRulesSupport(IndentationRules indentationRules) { + this._indentationRules = indentationRules; + } + + public boolean shouldIncrease(final String text) { + return _indentationRules.increaseIndentPattern.matchesFully(text); + + // if (this._indentationRules.indentNextLinePattern && this._indentationRules.indentNextLinePattern.test(text)) { + // return true; + // } + } + + public boolean shouldDecrease(final String text) { + return _indentationRules.decreaseIndentPattern.matchesFully(text); + } + + public boolean shouldIndentNextLine(final String text) { + return this._indentationRules.indentNextLinePattern != null && this._indentationRules.indentNextLinePattern.matchesFully(text); + } + + public boolean shouldIgnore(final String text) { + // the text matches `unIndentedLinePattern` + return this._indentationRules.unIndentedLinePattern != null && this._indentationRules.unIndentedLinePattern.matchesFully(text); + } + + public int getIndentMetadata(final String text) { + int ret = 0; + if (this.shouldIncrease(text)) { + ret += IndentConsts.INCREASE_MASK; + } + if (this.shouldDecrease(text)) { + ret += IndentConsts.DECREASE_MASK; + } + if (this.shouldIndentNextLine(text)) { + ret += IndentConsts.INDENT_NEXTLINE_MASK; + } + if (this.shouldIgnore(text)) { + ret += IndentConsts.UNINDENT_MASK; + } + return ret; + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/OnEnterSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/OnEnterSupport.java new file mode 100644 index 000000000..f4b0830c9 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/supports/OnEnterSupport.java @@ -0,0 +1,139 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Angelo Zerr - initial API and implementation + * Sebastian Thomschke (Vegard IT GmbH) - add previousLineText support + */ +package org.eclipse.tm4e.languageconfiguration.internal.supports; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.languageconfiguration.internal.model.CharacterPair; +import org.eclipse.tm4e.languageconfiguration.internal.model.EnterAction; +import org.eclipse.tm4e.languageconfiguration.internal.model.EnterAction.IndentAction; +import org.eclipse.tm4e.languageconfiguration.internal.model.OnEnterRule; +import org.eclipse.tm4e.languageconfiguration.internal.utils.RegExpUtils; + +/** + * On enter support. + * + * @see + * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/onEnter.ts + */ +public class OnEnterSupport { + + private static final List DEFAULT_BRACKETS = List.of( + new CharacterPair("(", ")"), //$NON-NLS-1$ //$NON-NLS-2$ + new CharacterPair("{", "}"), //$NON-NLS-1$ //$NON-NLS-2$ + new CharacterPair("[", "]")); //$NON-NLS-1$ //$NON-NLS-2$ + + private final List brackets; + private final List regExpRules; + + public OnEnterSupport(@Nullable final List brackets, @Nullable final List regExpRules) { + this.brackets = (brackets != null ? brackets : DEFAULT_BRACKETS) + .stream() + .filter(Objects::nonNull) + .map(ProcessedBracketPair::new) + .collect(Collectors.toList()); + + this.regExpRules = regExpRules != null ? regExpRules : Collections.emptyList(); + } + + public @Nullable EnterAction onEnter( + // TODO autoIndent: EditorAutoIndentStrategy, + final String previousLineText, + final String beforeEnterText, + final String afterEnterText) { + // (1): `regExpRules` + // if (autoIndent >= EditorAutoIndentStrategy.Advanced) { + for (final OnEnterRule rule : regExpRules) { + if (!rule.beforeText.matchesPartially(beforeEnterText)) + continue; + + final var afterTextPattern = rule.afterText; + if (afterTextPattern != null && !afterTextPattern.matchesPartially(afterEnterText)) + continue; + + final var previousLinePattern = rule.previousLineText; + if (previousLinePattern != null && !previousLinePattern.matchesPartially(previousLineText)) + continue; + + return rule.action; + } + + // (2): Special indent-outdent + // if (autoIndent >= EditorAutoIndentStrategy.Brackets) { + if (!beforeEnterText.isEmpty() && !afterEnterText.isEmpty()) { + for (final ProcessedBracketPair bracket : brackets) { + if (bracket.matchOpen(beforeEnterText) && bracket.matchClose(afterEnterText)) { + return new EnterAction(IndentAction.IndentOutdent); + } + } + } + + // (3): Open bracket based logic + // if (autoIndent >= EditorAutoIndentStrategy.Brackets) { + if (!beforeEnterText.isEmpty()) { + for (final ProcessedBracketPair bracket : brackets) { + if (bracket.matchOpen(beforeEnterText)) { + return new EnterAction(IndentAction.Indent); + } + } + } + + return null; + } + + private static final class ProcessedBracketPair { + + private static final Pattern B_REGEXP = Pattern.compile("\\B"); //$NON-NLS-1$ + + private final @Nullable Pattern openRegExp; + private final @Nullable Pattern closeRegExp; + + private ProcessedBracketPair(final CharacterPair charPair) { + openRegExp = createOpenBracketRegExp(charPair.open); + closeRegExp = createCloseBracketRegExp(charPair.close); + } + + private boolean matchOpen(final String beforeEnterText) { + return openRegExp != null && openRegExp.matcher(beforeEnterText).find(); + } + + private boolean matchClose(final String afterEnterText) { + return closeRegExp != null && closeRegExp.matcher(afterEnterText).find(); + } + + private static @Nullable Pattern createOpenBracketRegExp(final String bracket) { + final var str = new StringBuilder(RegExpUtils.escapeRegExpCharacters(bracket)); + final var c = String.valueOf(str.charAt(0)); + if (!B_REGEXP.matcher(c).find()) { + str.insert(0, "\\b"); //$NON-NLS-1$ + } + str.append("\\s*$"); //$NON-NLS-1$ + return RegExpUtils.create(str.toString()); + } + + private static @Nullable Pattern createCloseBracketRegExp(final String bracket) { + final var str = new StringBuilder(RegExpUtils.escapeRegExpCharacters(bracket)); + final var c = String.valueOf(str.charAt(str.length() - 1)); + if (!B_REGEXP.matcher(c).find()) { + str.append("\\b"); //$NON-NLS-1$ + } + str.insert(0, "^\\s*"); //$NON-NLS-1$ + return RegExpUtils.create(str.toString()); + } + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/RegExpUtils.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/RegExpUtils.java new file mode 100644 index 000000000..5d8afd00d --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/RegExpUtils.java @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2015-2017 Angelo ZERR. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + *

+ * SPDX-License-Identifier: EPL-2.0 + *

+ * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.utils; + + +import java.util.regex.Pattern; + +import org.eclipse.jdt.annotation.Nullable; + +import io.github.rosemoe.sora.util.Logger; + +/** + * Regex utilities. + */ +public final class RegExpUtils { + + private static final Logger log = Logger.instance(RegExpUtils.class.getName()); + + /** + * Escapes regular expression characters in a given string + */ + public static String escapeRegExpCharacters(final String value) { + return value.replaceAll("[\\-\\\\\\{\\}\\*\\+\\?\\|\\^\\$\\.\\[\\]\\(\\)\\#]", "\\\\$0"); //$NON-NLS-1$ //$NON-NLS-2$ + } + + /** + * Create Java Regexp and null otherwise. + * + * @return Java Regexp and null otherwise. + */ + @Nullable + public static Pattern create(final String regex) { + try { + return Pattern.compile(regex); + } catch (final Exception ex) { + log.e("Failed to parse pattern: " + regex, ex); + return null; + } + } + + private RegExpUtils() { + } +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/TextUtils.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/TextUtils.java new file mode 100644 index 000000000..533878e11 --- /dev/null +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/utils/TextUtils.java @@ -0,0 +1,216 @@ +/** + * Copyright (c) 2015-2022 Angelo ZERR and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + *

+ * SPDX-License-Identifier: EPL-2.0 + *

+ * Contributors: + * Angelo Zerr - initial API and implementation + */ +package org.eclipse.tm4e.languageconfiguration.internal.utils; + +import io.github.rosemoe.sora.text.CharPosition; +import io.github.rosemoe.sora.text.Content; +import io.github.rosemoe.sora.util.IntPair; + +public final class TextUtils { + + /** + * @return true if text of the command is an enter and false otherwise. + */ + /*public static boolean isEnter(final IDocument doc, final DocumentCommand cmd) { + return cmd.length == 0 && cmd.text != null && TextUtilities.equals(doc.getLegalLineDelimiters(), cmd.text) != -1; + }*/ + public static String normalizeIndentation(final String text, final int tabSize, final boolean insertSpaces) { + int firstNonWhitespaceIndex = firstNonWhitespaceIndex(text); + if (firstNonWhitespaceIndex == -1) { + firstNonWhitespaceIndex = text.length(); + } + return normalizeIndentationFromWhitespace(text.substring(0, firstNonWhitespaceIndex), tabSize, insertSpaces) + + text.substring(firstNonWhitespaceIndex); + } + + private static String normalizeIndentationFromWhitespace(final String text, final int tabSize, final boolean insertSpaces) { + int spacesCnt = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '\t') { + spacesCnt += tabSize; + } else { + spacesCnt++; + } + } + + final var result = new StringBuilder(); + if (!insertSpaces) { + final long tabsCnt = spacesCnt / tabSize; + spacesCnt = spacesCnt % tabSize; + for (int i = 0; i < tabsCnt; i++) { + result.append('\t'); + } + } + + for (int i = 0; i < spacesCnt; i++) { + result.append(' '); + } + + return result.toString(); + } + + /** + * Returns the start of the string at the offset in the text. If the string is not in the text at the offset, returns -1. + *

+ * Example: + * + * 

+     * text="apple banana", offset=8, string="banana" -> returns 6
+     *
+ */ + public static int startIndexOfOffsetTouchingString(final String text, final int offset, final String string) { + int start = offset - string.length(); + start = start < 0 ? 0 : start; + int end = offset + string.length(); + end = end >= text.length() ? text.length() : end; + try { + final int indexInSubtext = text.substring(start, end).indexOf(string); + return indexInSubtext == -1 ? -1 : start + indexInSubtext; + } catch (final IndexOutOfBoundsException e) { + return -1; + } + } + + /** + * Returns first index of the string that is not whitespace. If string is empty + * or contains only whitespaces, returns -1 + */ + private static int firstNonWhitespaceIndex(final String text) { + for (int i = 0, len = text.length(); i < len; i++) { + final char c = text.charAt(i); + if (c != ' ' && c != '\t') { + return i; + } + } + return -1; + } + + public static String getIndentationFromWhitespace(final String whitespace, final int tabSize, final boolean insertSpaces) { + final var tab = "\t"; //$NON-NLS-1$ + int indentOffset = 0; + boolean startsWithTab = true; + boolean startsWithSpaces = true; + final String spaces = insertSpaces + ? " ".repeat(tabSize) + : ""; + while (startsWithTab || startsWithSpaces) { + startsWithTab = whitespace.startsWith(tab, indentOffset); + startsWithSpaces = insertSpaces && whitespace.startsWith(spaces, indentOffset); + if (startsWithTab) { + indentOffset += tab.length(); + } + if (startsWithSpaces) { + indentOffset += spaces.length(); + } + } + return whitespace.substring(0, indentOffset); + } + + public static String getIndentationAtPosition(final Content doc, final int offset) { + try { + // find start offset of current line + final int lineStartOffset = doc.getIndexer().getCharPosition(offset).index; + + // find white spaces + final int indentationEndOffset = findEndOfWhiteSpace(doc, lineStartOffset, offset); + + return doc.substring(lineStartOffset, indentationEndOffset - lineStartOffset); + } catch (final Exception excp) { + // stop work + } + return ""; //$NON-NLS-1$ + } + + /** + * Returns the first offset greater than offset and smaller than + * end whose character is not a space or tab character. If no such + * offset is found, end is returned. + * + * @param doc the document to search in + * @param startAt the offset at which searching start + * @param endAt the offset at which searching stops + * @return the offset in the specified range whose character is not a space or tab + */ + private static int findEndOfWhiteSpace(final Content doc, int startAt, final int endAt) { + while (startAt < endAt) { + final char c = doc.charAt(startAt); + if (c != ' ' && c != '\t') { + return startAt; + } + startAt++; + } + return endAt; + } + + /** + * Determines if all the characters at any offset of the specified document line are the whitespace characters. + * + * @param doc the document to search in + * @param line zero-based document line number + * @return true if all the characters of the specified document line are the whitespace + * characters, otherwise returns false + */ + public static boolean isBlankLine(final Content doc, final int line) { + try { + int offset = doc.charAt(line, 0); + final int lineEnd = offset + doc.getLine(line).length(); + while (offset < lineEnd) { + if (!Character.isWhitespace(doc.charAt(offset))) { + return false; + } + offset++; + } + } catch (final Exception e) { + // Ignore, forcing a positive result + } + return true; + } + + /** + * Returns the leading whitespace of the string. + * If the string contains only whitespaces, returns entire string + */ + public static String getLeadingWhitespace(String str, int start, int end) { + for (var i = start; i < end; i++) { + var chCode = str.charAt(i); + if (chCode != 32 /*CharCode.Space*/ && chCode != 9/*CharCode.Tab*/) { + return str.substring(start, i); + } + } + return str.substring(start, end); + } + + public static String getLinePrefixingWhitespaceAtPosition(final Content d, final CharPosition position) { + + var line = d.getLine(position.line); + + var startIndex = IntPair.getFirst(io.github.rosemoe.sora.text.TextUtils.findLeadingAndTrailingWhitespacePos( + line + )); + + return line.subSequence(0, startIndex).toString(); + + } + + /** + * Returns the leading whitespace of the string. + * If the string contains only whitespaces, returns entire string + */ + public static String getLeadingWhitespace(String str) { + return getLeadingWhitespace(str, 0, str.length()); + } + + + private TextUtils() { + } + +} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPair.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPair.java deleted file mode 100644 index 4c3079053..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPair.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -/** - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L196 - */ -public class AutoClosingPair extends CharacterPair { - - public AutoClosingPair(final String open, final String close) { - super(open, close); - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPairConditional.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPairConditional.java deleted file mode 100644 index e476615eb..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/AutoClosingPairConditional.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import java.util.List; - -/** - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L201 - */ -public final class AutoClosingPairConditional extends AutoClosingPair { - - public /*final*/ List notIn; - - public AutoClosingPairConditional(final String open, final String close, final List notIn) { - super(open, close); - this.notIn = notIn; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CharacterPair.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CharacterPair.java deleted file mode 100644 index f4e3e3e04..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CharacterPair.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import java.util.Objects; - -/** - * A tuple of two characters, like a pair of opening and closing brackets. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L194 - */ -public class CharacterPair { - - public final String open; - public final String close; - - public CharacterPair(final String opening, final String closing) { - this.open = opening; - this.close = closing; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - CharacterPair that = (CharacterPair) o; - - if (!Objects.equals(open, that.open)) return false; - return Objects.equals(close, that.close); - } - - @Override - public int hashCode() { - int result = open != null ? open.hashCode() : 0; - result = 31 * result + (close != null ? close.hashCode() : 0); - return result; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CommentRule.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CommentRule.java deleted file mode 100644 index 3754ca025..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CommentRule.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import org.eclipse.jdt.annotation.Nullable; - -/** - * Describes how comments for a language work. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L13 - */ -public final class CommentRule { - - /** - * The line comment token, like `// this is a comment` - */ - @Nullable - public final String lineComment; - - /** - * The block comment character pair, like `/* block comment */` - */ - @Nullable - public final CharacterPair blockComment; - - public CommentRule(@Nullable final String lineComment, @Nullable final CharacterPair blockComment) { - this.lineComment = lineComment; - this.blockComment = blockComment; - } -} \ No newline at end of file diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CompleteEnterAction.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CompleteEnterAction.java deleted file mode 100644 index 5f64d7fce..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/CompleteEnterAction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -/** - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L250 - */ -public final class CompleteEnterAction extends EnterAction { - - /** - * The line's indentation minus removeText - */ - public final String indentation; - - public CompleteEnterAction(final EnterAction action, final String indentation) { - super(action.indentAction); - this.indentation = indentation; - this.appendText = action.appendText; - this.removeText = action.removeText; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/EnterAction.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/EnterAction.java deleted file mode 100644 index 2636181bd..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/EnterAction.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import org.eclipse.jdt.annotation.Nullable; - -/** - * Describes what to do when pressing Enter. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L232 - */ -public class EnterAction { - - - - public enum IndentAction { - /** - * Insert new line and copy the previous line's indentation. - */ - None, - /** - * Insert new line and indent once (relative to the previous line's - * indentation). - */ - Indent, - /** - * Insert two new lines: - the first one indented which will hold the cursor - - * the second one at the same indentation level - */ - IndentOutdent, - /** - * Insert new line and outdent once (relative to the previous line's - * indentation). - */ - Outdent; - } - - /** - * Describe what to do with the indentation. - */ - public final IndentAction indentAction; - - /** - * Describes text to be appended after the new line and after the indentation. - */ - @Nullable - public String appendText; - - /** - * Describes the number of characters to remove from the new line's indentation. - */ - @Nullable - public Integer removeText; - - public EnterAction(final IndentAction indentAction) { - this.indentAction = indentAction; - } - - /** - * @param appendText the appendText to set - */ - EnterAction withAppendText(@Nullable final String appendText) { - this.appendText = appendText; - return this; - } - - /** - * @param removeText the removeText to set - */ - EnterAction withRemoveText(@Nullable final Integer removeText) { - this.removeText = removeText; - return this; - } - - public EnterAction copy() { - var copy = new EnterAction(indentAction); - copy.appendText = appendText; - copy.removeText = removeText; - return copy; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/FoldingRules.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/FoldingRules.java deleted file mode 100644 index 497f3360d..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/FoldingRules.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import java.util.regex.Pattern; - -/** - * Describes folding rules for a language. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L139 - */ -public final class FoldingRules { - - /** - * Used by the indentation based strategy to decide whether empty lines belong to the previous or the next block. - * A language adheres to the off-side rule if blocks in that language are expressed by their indentation. - * See [wikipedia](https://en.wikipedia.org/wiki/Off-side_rule) for more information. - */ - public final boolean offSide; - public final Pattern markersStart; - public final Pattern markersEnd; - - public FoldingRules(final boolean offSide, final Pattern markersStart, final Pattern markersEnd) { - this.offSide = offSide; - this.markersStart = markersStart; - this.markersEnd = markersEnd; - } -} \ No newline at end of file diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/IndentationRules.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/IndentationRules.java deleted file mode 100644 index 8c9ce98ce..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/IndentationRules.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import org.eclipse.jdt.annotation.Nullable; - -import java.util.regex.Pattern; - -/** - * Describes indentation rules for a language. - * - * @see - * https://github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration - */ -public class IndentationRules { - - /** - * If a line matches this pattern, then all the lines after it should be unindented once (until another rule matches). - */ - public final Pattern decreaseIndentPattern; - /** - * If a line matches this pattern, then all the lines after it should be indented once (until another rule matches). - */ - public final Pattern increaseIndentPattern; - /** - * If a line matches this pattern, then **only the next line** after it should be indented once. - */ - @Nullable - public final Pattern indentNextLinePattern; - /** - * If a line matches this pattern, then its indentation should not be changed and it should not be evaluated against the other rules. - */ - @Nullable - public final Pattern unIndentedLinePattern; - - - public IndentationRules(Pattern decreaseIndentPattern, Pattern increaseIndentPattern, @Nullable Pattern indentNextLinePattern, @Nullable Pattern unIndentedLinePattern) { - this.decreaseIndentPattern = decreaseIndentPattern; - this.increaseIndentPattern = increaseIndentPattern; - this.indentNextLinePattern = indentNextLinePattern; - this.unIndentedLinePattern = unIndentedLinePattern; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/LanguageConfiguration.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/LanguageConfiguration.java deleted file mode 100644 index a38ca275f..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/LanguageConfiguration.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import java.io.BufferedReader; -import java.io.Reader; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -import org.eclipse.jdt.annotation.NonNull; -import org.eclipse.jdt.annotation.NonNullByDefault; -import org.eclipse.jdt.annotation.Nullable; -import org.eclipse.tm4e.core.internal.oniguruma.OnigRegExp; -import org.eclipse.tm4e.languageconfiguration.model.EnterAction.IndentAction; -import org.eclipse.tm4e.languageconfiguration.utils.RegExpUtils; -import org.eclipse.tm4e.languageconfiguration.utils.TextUtils; - -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; - -/** - * The language configuration interface defines the contract between extensions and various editor features, like - * automatic bracket insertion, automatic indentation etc. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts - */ -public class LanguageConfiguration { - - /** - * See JSON format at https://code.visualstudio.com/api/language-extensions/language-configuration-guide - * - * @return an instance of {@link LanguageConfiguration} loaded from the VSCode language-configuration.json file - * reader. - */ - @NonNullByDefault({}) - @Nullable - public static LanguageConfiguration load(@NonNull final Reader reader) { - return new GsonBuilder() - - .registerTypeAdapter(OnEnterRule.class, (JsonDeserializer) (json, typeOfT, context) -> { - if (!json.isJsonObject()) { - return null; - } - - final var jsonObj = json.getAsJsonObject(); - final var beforeText = getAsPattern(jsonObj.get("beforeText")); //$NON-NLS-1$ - if (beforeText == null) { - return null; - } - - final var actionElem = jsonObj.get("action"); //$NON-NLS-1$ - if (actionElem != null && actionElem.isJsonObject()) { - final var actionJsonObj = actionElem.getAsJsonObject(); - var action1 = actionJsonObj.get("indentAction"); - var action2 = actionJsonObj.get("indent"); - final var indentActionString = getAsString(action1 == null ? action2 : action1); //$NON-NLS-1$ - if (indentActionString != null) { - final var afterText = getAsPattern(jsonObj.get("afterText")); //$NON-NLS-1$ - final var indentAction = IndentAction.valueOf( - TextUtils.firstCharToUpperCase(indentActionString)); - final var removeText = getAsInteger(actionJsonObj.get("removeText")); //$NON-NLS-1$ - final var appendText = getAsString(actionJsonObj.get("appendText")); - - final var previousLineText = getAsPattern(actionJsonObj.get("previousLineText")); - - //$NON-NLS-1$ - final var action = new EnterAction(indentAction); - action.appendText = appendText; - action.removeText = removeText; - return new OnEnterRule(beforeText, afterText, previousLineText, action); - } - } - return null; - }) - - .registerTypeAdapter(CommentRule.class, (JsonDeserializer) (json, typeOfT, context) -> { - if (!json.isJsonObject()) { - return null; - } - - // ex: {"lineComment": "//","blockComment": [ "/*", "*/" ]} - final var jsonObj = json.getAsJsonObject(); - final var lineComment = getAsString(jsonObj.get("lineComment")); //$NON-NLS-1$ - final var blockCommentElem = jsonObj.get("blockComment"); //$NON-NLS-1$ - CharacterPair blockComment = null; - if (blockCommentElem != null && blockCommentElem.isJsonArray()) { - final var blockCommentArray = blockCommentElem.getAsJsonArray(); - if (blockCommentArray.size() == 2) { - final var blockCommentStart = getAsString(blockCommentArray.get(0)); - final var blockCommentEnd = getAsString(blockCommentArray.get(1)); - if (blockCommentStart != null && blockCommentEnd != null) { - blockComment = new CharacterPair(blockCommentStart, blockCommentEnd); - } - } - } - - return lineComment == null && blockComment == null - ? null - : new CommentRule(lineComment, blockComment); - }) - - .registerTypeAdapter(CharacterPair.class, (JsonDeserializer) (json, typeOfT, - context) -> { - if (!json.isJsonArray()) { - return null; - } - - // ex: ["{","}"] - final var charsPair = json.getAsJsonArray(); - if (charsPair.size() != 2) { - return null; - } - - final var open = getAsString(charsPair.get(0)); - final var close = getAsString(charsPair.get(1)); - - return open == null || close == null - ? null - : new CharacterPair(open, close); - }) - - .registerTypeAdapter(AutoClosingPair.class, (JsonDeserializer) (json, typeOfT, - context) -> { - String open = null; - String close = null; - if (json.isJsonArray()) { - // ex: ["{","}"] - final var charsPair = json.getAsJsonArray(); - if (charsPair.size() != 2) { - return null; - } - open = getAsString(charsPair.get(0)); - close = getAsString(charsPair.get(1)); - } else if (json.isJsonObject()) { - // ex: {"open":"'","close":"'", "notIn": ["string", "comment"]} - final var autoClosePair = json.getAsJsonObject(); - open = getAsString(autoClosePair.get("open")); //$NON-NLS-1$ - close = getAsString(autoClosePair.get("close")); //$NON-NLS-1$ - } - - return open == null || close == null - ? null - : new AutoClosingPair(open, close); - }) - - .registerTypeAdapter(AutoClosingPairConditional.class, (JsonDeserializer) ( - json, typeOfT, context) -> { - final var notInList = new ArrayList(2); - String open = null; - String close = null; - if (json.isJsonArray()) { - // ex: ["{","}"] - final var charsPair = json.getAsJsonArray(); - if (charsPair.size() != 2) { - return null; - } - open = getAsString(charsPair.get(0)); - close = getAsString(charsPair.get(1)); - } else if (json.isJsonObject()) { - // ex: {"open":"'","close":"'", "notIn": ["string", "comment"]} - final var autoClosePair = json.getAsJsonObject(); - open = getAsString(autoClosePair.get("open")); //$NON-NLS-1$ - close = getAsString(autoClosePair.get("close")); //$NON-NLS-1$ - final var notInElem = autoClosePair.get("notIn"); //$NON-NLS-1$ - if (notInElem != null && notInElem.isJsonArray()) { - for (JsonElement element : notInElem.getAsJsonArray()) { - final var string = getAsString(element); - if (string != null) { - notInList.add(string); - } - } - } - } - - return open == null || close == null - ? null - : new AutoClosingPairConditional(open, close, notInList); - }) - - .registerTypeAdapter(IndentationRules.class, (JsonDeserializer) (json, typeT, context) -> { - - if (!json.isJsonObject()) { - return null; - } - - final var object = json.getAsJsonObject(); - - final var increaseIndentPattern = getAsPattern( - object.get("increaseIndentPattern")); - - final var decreaseIndentPattern = getAsPattern( - object.get("decreaseIndentPattern")); - - final var indentNextLinePattern = getAsPattern( - object.get("indentNextLinePattern")); - - final var unIndentedLinePattern = getAsPattern( - object.get("unIndentedLinePattern")); - - if (increaseIndentPattern == null || decreaseIndentPattern == null) { - return null; - } - - return new IndentationRules(decreaseIndentPattern, increaseIndentPattern, indentNextLinePattern, unIndentedLinePattern); - }) - - .registerTypeAdapter(FoldingRules.class, (JsonDeserializer) (json, typeOfT, context) -> { - if (!json.isJsonObject()) { - return null; - } - - // ex: {"offSide": true, "markers": {"start": "^\\s*/", "end": "^\\s*"}} - final var jsonObj = json.getAsJsonObject(); - final var markersElem = jsonObj.get("markers"); //$NON-NLS-1$ - if (markersElem != null && markersElem.isJsonObject()) { - final var offSide = getAsBoolean(jsonObj.get("offSide")); //$NON-NLS-1$ - final var markersObj = markersElem.getAsJsonObject(); - final var startMarker = getAsPattern(markersObj.get("start")); //$NON-NLS-1$ - final var endMarker = getAsPattern(markersObj.get("end")); //$NON-NLS-1$ - if (startMarker != null && endMarker != null) { - return new FoldingRules(offSide, startMarker, endMarker); - } - } - return null; - }) - .create() - .fromJson(new BufferedReader(reader), LanguageConfiguration.class); - } - - @Nullable - private static Pattern getAsPattern(@Nullable final JsonElement element) { - String pattern; - if (element != null && element.isJsonObject()) { - pattern = element.getAsJsonObject().get("pattern").getAsString(); - } else { - pattern = getAsString(element); - } - return pattern == null ? null : RegExpUtils.create(pattern); - } - - @Nullable - private static OnigRegExp getAsOnigRegExp(@Nullable final JsonElement element) { - String pattern; - if (element != null && element.isJsonObject()) { - pattern = element.getAsJsonObject().get("pattern").getAsString(); - } else { - pattern = getAsString(element); - } - return pattern == null ? null : new OnigRegExp(pattern); - } - - @Nullable - private static String getAsString(@Nullable final JsonElement element) { - if (element == null) { - return null; - } - try { - return element.getAsString(); - } catch (final Exception e) { - e.printStackTrace(); - return null; - } - } - - private static boolean getAsBoolean(@Nullable final JsonElement element) { - if (element == null) { - return false; - } - try { - return element.getAsBoolean(); - } catch (final Exception e) { - return false; - } - } - - @Nullable - private static Integer getAsInteger(@Nullable final JsonElement element) { - if (element == null) { - return null; - } - try { - return element.getAsInt(); - } catch (final Exception e) { - return null; - } - } - - @Nullable - private CommentRule comments; - - /** - * Returns the language's comments. The comments are used by {@link AutoClosingPairConditional} when - * notIn contains comment - * - * @return the language's comments. - */ - @Nullable - public CommentRule getComments() { - return comments; - } - - @Nullable - private List brackets; - - /** - * Returns the language's brackets. This configuration implicitly affects pressing Enter around these brackets. - * - * @return the language's brackets - */ - @Nullable - public List getBrackets() { - return brackets; - } - - @Nullable - private String wordPattern; - - /** - * Returns the language's definition of a word. This is the regex used when referring to a word. - * - * @return the language's word pattern. - */ - @Nullable - public String getWordPattern() { - return wordPattern; - } - - @Nullable - private List onEnterRules; - - /** - * Returns the language's rules to be evaluated when pressing Enter. - * - * @return the language's rules to be evaluated when pressing Enter. - */ - @Nullable - public List getOnEnterRules() { - return onEnterRules; - } - - @Nullable - private List autoClosingPairs; - - /** - * Returns the language's auto closing pairs. The 'close' character is automatically inserted with the 'open' - * character is typed. If not set, the configured brackets will be used. - * - * @return the language's auto closing pairs. - */ - @Nullable - public List getAutoClosingPairs() { - return autoClosingPairs; - } - - @Nullable - private List surroundingPairs; - - /** - * Returns the language's surrounding pairs. When the 'open' character is typed on a selection, the selected string - * is surrounded by the open and close characters. If not set, the autoclosing pairs settings will be used. - * - * @return the language's surrounding pairs. - */ - @Nullable - public List getSurroundingPairs() { - return surroundingPairs; - } - - // TODO @Nullable List getColorizedBracketPairs(); - - - private IndentationRules indentationRules; - - /** - * The language's indentation settings. - */ - @Nullable - public IndentationRules getIndentationRules() { - return indentationRules; - } - - @Nullable - private String autoCloseBefore; - - @Nullable - public String getAutoCloseBefore() { - return autoCloseBefore; - } - - @Nullable - private FoldingRules folding; - - /** - * Returns the language's folding rules. - * - * @return the language's folding rules. - */ - @Nullable - public FoldingRules getFolding() { - return folding; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/OnEnterRule.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/OnEnterRule.java deleted file mode 100644 index 89b48bf8e..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/model/OnEnterRule.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.model; - -import java.util.regex.Pattern; - -import org.eclipse.jdt.annotation.Nullable; - -/** - * Describes a rule to be evaluated when pressing Enter. - * - * @see - * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/languageConfiguration.ts#L157 - */ -public final class OnEnterRule { - /** - * This rule will only execute if the text before the cursor matches this regular expression. - */ - public final Pattern beforeText; - - /** - * This rule will only execute if the text after the cursor matches this regular expression. - */ - @Nullable - public final Pattern afterText; - - /** - * This rule will only execute if the text above the this line matches this regular expression. - */ - @Nullable - public final Pattern previousLineText; - - - /** - * The action to execute. - */ - public final EnterAction action; - - public OnEnterRule(final Pattern beforeText, @Nullable final Pattern afterText, - @Nullable final Pattern previousLineText, final EnterAction action) { - this.beforeText = beforeText; - this.afterText = afterText; - this.previousLineText = previousLineText; - this.action = action; - } - -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CharacterPairSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CharacterPairSupport.java deleted file mode 100644 index 1b0aa0be6..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CharacterPairSupport.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.supports; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import org.eclipse.jdt.annotation.Nullable; -import org.eclipse.tm4e.languageconfiguration.model.AutoClosingPair; -import org.eclipse.tm4e.languageconfiguration.model.AutoClosingPairConditional; -import org.eclipse.tm4e.languageconfiguration.model.LanguageConfiguration; - -/** - * The "character pair" support. - * - * @see - * https://github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/characterPair.ts - */ -public final class CharacterPairSupport { - - public static final String DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED = ";:.,=}])> \r\n\t"; - public static final String DEFAULT_AUTOCLOSE_BEFORE_WHITESPACE = " \r\n\t"; - - public final List autoClosingPairs; - public final List surroundingPairs; - public final String autoCloseBefore; - - @SuppressWarnings("unchecked") - public CharacterPairSupport(LanguageConfiguration config) { - final var autoClosingPairs = config.getAutoClosingPairs(); - final var brackets = config.getBrackets(); - - if (autoClosingPairs != null) { - this.autoClosingPairs = autoClosingPairs.stream().filter(Objects::nonNull) - .map(el -> new AutoClosingPairConditional(el.open, el.close, el.notIn)) - .collect(Collectors.toList()); - } else if (brackets != null) { - this.autoClosingPairs = brackets.stream().filter(Objects::nonNull) - .map(el -> new AutoClosingPairConditional(el.open, el.close, Collections.emptyList())) - .collect(Collectors.toList()); - } else { - this.autoClosingPairs = Collections.emptyList(); - } - - final var autoCloseBefore = config.getAutoCloseBefore(); - this.autoCloseBefore = autoCloseBefore != null - ? autoCloseBefore - : CharacterPairSupport.DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED; - - final var surroundingPairs = config.getSurroundingPairs(); - this.surroundingPairs = surroundingPairs != null - ? surroundingPairs.stream().filter(Objects::nonNull).collect(Collectors.toList()) - : (List) (List) this.autoClosingPairs; - } - - /** - * TODO not declared in upstream project - */ - @Nullable - public AutoClosingPairConditional getAutoClosingPair(final String text, final int offset, - final String newCharacter) { - if (newCharacter.isEmpty()) { - return null; - } - for (final AutoClosingPairConditional autoClosingPair : autoClosingPairs) { - final String opening = autoClosingPair.open; - if (!opening.endsWith(newCharacter)) { - continue; - } - if (opening.length() > 1) { - final String offsetPrefix = text.substring(0, offset); - if (!offsetPrefix.endsWith(opening.substring(0, opening.length() - 1))) { - continue; - } - } - return autoClosingPair; - } - return null; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CommentSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CommentSupport.java deleted file mode 100644 index 5016ae0bf..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/CommentSupport.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.supports; - -import org.eclipse.jdt.annotation.Nullable; - -import org.eclipse.tm4e.languageconfiguration.model.CharacterPair; -import org.eclipse.tm4e.languageconfiguration.model.CommentRule; - -import io.github.rosemoe.sora.text.Content; - -public final class CommentSupport { - - @Nullable - private final CommentRule comments; - - public CommentSupport(@Nullable final CommentRule comments) { - this.comments = comments; - } - - private boolean isInComment(final Content text, final int offset) { - try { - if (isInBlockComment(text.subSequence(0, offset).toString())) { - return true; - } - var indexer = text.getIndexer(); - final int line = indexer.getCharLine(offset); - final int lineOffset = indexer.getCharIndex(line,0); - return isInLineComment(text.subSequence(lineOffset, offset - lineOffset).toString()); - } catch (final Exception e) { - return false; - } - } - - @Nullable - public String getLineComment() { - final var comments = this.comments; - return comments == null ? null : comments.lineComment; - } - - @Nullable - public CharacterPair getBlockComment() { - final var comments = this.comments; - return comments == null ? null : comments.blockComment; - } - - private boolean isInLineComment(final String indexLinePrefix) { - final var comments = this.comments; - if (comments == null) - return false; - return indexLinePrefix.indexOf(comments.lineComment) != -1; - } - - private boolean isInBlockComment(final String indexPrefix) { - final var comments = this.comments; - if (comments == null) - return false; - - final var blockComment = comments.blockComment; - if (blockComment == null) - return false; - - final String commentOpen = blockComment.open; - final String commentClose = blockComment.close; - int index = indexPrefix.indexOf(commentOpen); - while (index != -1 && index < indexPrefix.length()) { - final int closeIndex = indexPrefix.indexOf(commentClose, index + commentOpen.length()); - if (closeIndex == -1) { - return true; - } - index = indexPrefix.indexOf(commentOpen, closeIndex + commentClose.length()); - } - return false; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/IndentRulesSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/IndentRulesSupport.java deleted file mode 100644 index a5e91165e..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/IndentRulesSupport.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.supports; - -import org.eclipse.tm4e.languageconfiguration.model.IndentationRules; -import org.eclipse.tm4e.languageconfiguration.utils.TextUtils; - -/** - * The "IndentRules" support. - * - * @see - * https://github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/indentRules.ts - */ -public class IndentRulesSupport { - - private final IndentationRules indentationRules; - - public IndentRulesSupport(IndentationRules indentationRules) { - this.indentationRules = indentationRules; - } - - public boolean shouldIncrease(String text) { - if (this.indentationRules.increaseIndentPattern == null || TextUtils.isEmpty(text)) { - return false; - } - return indentationRules.increaseIndentPattern.matcher(text).matches(); - - // if (this._indentationRules.indentNextLinePattern && this._indentationRules.indentNextLinePattern.test(text)) { - // return true; - // } - } - - public boolean shouldDecrease(String text) { - if (this.indentationRules.decreaseIndentPattern == null || TextUtils.isEmpty(text)) { - return false; - } - return indentationRules.decreaseIndentPattern.matcher(text).matches(); - } - - public boolean shouldIndentNextLine(String text) { - if (this.indentationRules.indentNextLinePattern == null || TextUtils.isEmpty(text)) { - return false; - } - return indentationRules.indentNextLinePattern.matcher(text).matches(); - } - - - public boolean shouldIgnore(String text) { - if (this.indentationRules.unIndentedLinePattern == null || TextUtils.isEmpty(text)) { - return false; - } - return indentationRules.unIndentedLinePattern.matcher(text).matches(); - } - - - public int getIndentMetadata(String text) { - int ret = 0; - if (this.shouldIncrease(text)) { - ret += IndentConsts.INCREASE_MASK; - } - if (this.shouldDecrease(text)) { - ret += IndentConsts.DECREASE_MASK; - } - if (this.shouldIndentNextLine(text)) { - ret += IndentConsts.INDENT_NEXTLINE_MASK; - } - if (this.shouldIgnore(text)) { - ret += IndentConsts.UNINDENT_MASK; - } - return ret; - } - - - public static class IndentConsts { - public static final int INCREASE_MASK = 0b00000001; - public static final int DECREASE_MASK = 0b00000010; - public static final int INDENT_NEXTLINE_MASK = 0b00000100; - public static final int UNINDENT_MASK = 0b00001000; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/OnEnterSupport.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/OnEnterSupport.java deleted file mode 100644 index e85b1ecdb..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/supports/OnEnterSupport.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.supports; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.eclipse.jdt.annotation.Nullable; -import org.eclipse.tm4e.languageconfiguration.model.CharacterPair; -import org.eclipse.tm4e.languageconfiguration.model.EnterAction; -import org.eclipse.tm4e.languageconfiguration.model.OnEnterRule; -import org.eclipse.tm4e.languageconfiguration.model.EnterAction.IndentAction; -import org.eclipse.tm4e.languageconfiguration.utils.RegExpUtils; - -/** - * On enter support. - * - * @see - * https://github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/onEnter.ts - */ -public class OnEnterSupport { - - private static final List DEFAULT_BRACKETS = List.of( - new CharacterPair("(", ")"), //$NON-NLS-1$ //$NON-NLS-2$ - new CharacterPair("{", "}"), //$NON-NLS-1$ //$NON-NLS-2$ - new CharacterPair("[", "]")); //$NON-NLS-1$ //$NON-NLS-2$ - - private final List brackets; - private final List regExpRules; - - public OnEnterSupport(@Nullable final List brackets, @Nullable final List regExpRules) { - this.brackets = (brackets != null ? brackets : DEFAULT_BRACKETS) - .stream() - .filter(Objects::nonNull) - .map(ProcessedBracketPair::new) - .collect(Collectors.toList()); - - this.regExpRules = regExpRules != null ? regExpRules : Collections.emptyList(); - } - - @Nullable - public EnterAction onEnter( - // TODO autoIndent: EditorAutoIndentStrategy, - final String previousLineText, - final String beforeEnterText, - final String afterEnterText) { - // (1): `regExpRules` - for (final OnEnterRule rule : regExpRules) { - if (rule == null) { - continue; - } - var regResult = 1; - - final var beforeText = rule.beforeText; - - regResult &= beforeText.matcher(beforeEnterText).matches() ? 1 : 0; - - final var afterText = rule.afterText; - if (afterText != null) { - regResult &= afterText.matcher(afterEnterText).matches() ? 1 : 0; - } - - final var previousText = rule.previousLineText; - if (previousText != null) { - regResult &= previousText.matcher(previousLineText).matches() ? 1 : 0; - } - - if (regResult == 1) { - return rule.action.copy(); - } - } - - // (2): Special indent-outdent - if (!beforeEnterText.isEmpty() && !afterEnterText.isEmpty()) { - for (final ProcessedBracketPair bracket : brackets) { - if (bracket.matchOpen(beforeEnterText) && bracket.matchClose(afterEnterText)) { - return new EnterAction(IndentAction.IndentOutdent); - } - } - } - - // (3): Open bracket based logic - if (!beforeEnterText.isEmpty()) { - for (final ProcessedBracketPair bracket : brackets) { - if (bracket.matchOpen(beforeEnterText)) { - return new EnterAction(IndentAction.Indent); - } - } - } - - return null; - } - - private static final class ProcessedBracketPair { - - private static final Pattern B_REGEXP = Pattern.compile("\\B"); //$NON-NLS-1$ - - @Nullable - private final Pattern openRegExp; - - @Nullable - private final Pattern closeRegExp; - - private ProcessedBracketPair(final CharacterPair charPair) { - openRegExp = createOpenBracketRegExp(charPair.open); - closeRegExp = createCloseBracketRegExp(charPair.close); - } - - private boolean matchOpen(final String beforeEnterText) { - return openRegExp != null && openRegExp.matcher(beforeEnterText).find(); - } - - private boolean matchClose(final String afterEnterText) { - return closeRegExp != null && closeRegExp.matcher(afterEnterText).find(); - } - - @Nullable - private static Pattern createOpenBracketRegExp(final String bracket) { - final var str = new StringBuilder(RegExpUtils.escapeRegExpCharacters(bracket)); - final var c = String.valueOf(str.charAt(0)); - if (!B_REGEXP.matcher(c).find()) { - str.insert(0, "\\b"); //$NON-NLS-1$ - } - str.append("\\s*$"); //$NON-NLS-1$ - return RegExpUtils.create(str.toString()); - } - - @Nullable - private static Pattern createCloseBracketRegExp(final String bracket) { - final var str = new StringBuilder(RegExpUtils.escapeRegExpCharacters(bracket)); - final var c = String.valueOf(str.charAt(str.length() - 1)); - if (!B_REGEXP.matcher(c).find()) { - str.append("\\b"); //$NON-NLS-1$ - } - str.insert(0, "^\\s*"); //$NON-NLS-1$ - return RegExpUtils.create(str.toString()); - } - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/RegExpUtils.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/RegExpUtils.java deleted file mode 100644 index a0aeffb63..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/RegExpUtils.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.utils; - -import android.util.Log; - -import java.util.regex.Pattern; - -import org.eclipse.jdt.annotation.Nullable; - -/** - * Regex utilities. - */ -public final class RegExpUtils { - - /** - * Escapes regular expression characters in a given string - */ - public static String escapeRegExpCharacters(final String value) { - return value.replaceAll("[\\-\\\\\\{\\}\\*\\+\\?\\|\\^\\$\\.\\[\\]\\(\\)\\#]", "\\\\$0"); //$NON-NLS-1$ //$NON-NLS-2$ - } - - /** - * Create Java Regexp and null otherwise. - * - * @return Java Regexp and null otherwise. - */ - @Nullable - public static Pattern create(final String regex) { - try { - return Pattern.compile(regex); - } catch (final Exception ex) { - Log.e(RegExpUtils.class.getSimpleName(), "Failed to parse pattern: " + regex, ex); - return null; - } - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TabSpacesInfo.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TabSpacesInfo.java deleted file mode 100644 index 4405ef172..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TabSpacesInfo.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.utils; - -public final class TabSpacesInfo { - - private final int tabSize; - private final boolean insertSpaces; - - public TabSpacesInfo(final int tabSize, final boolean insertSpaces) { - this.tabSize = tabSize; - this.insertSpaces = insertSpaces; - } - - public int getTabSize() { - return tabSize; - } - - public boolean isInsertSpaces() { - return insertSpaces; - } -} diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TextUtils.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TextUtils.java deleted file mode 100644 index d7377ce51..000000000 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/utils/TextUtils.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * sora-editor - the awesome code editor for Android - * https://github.com/Rosemoe/sora-editor - * Copyright (C) 2020-2022 Rosemoe - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - * USA - * - * Please contact Rosemoe by email 2073412493@qq.com if you need - * additional information or have any questions - */ -package org.eclipse.tm4e.languageconfiguration.utils; - -import io.github.rosemoe.sora.langs.textmate.TextMateLanguage; -import io.github.rosemoe.sora.text.CharPosition; -import io.github.rosemoe.sora.text.Content; -import io.github.rosemoe.sora.util.IntPair; - -public class TextUtils { - - - public static String normalizeIndentation(final String str, final int tabSize, final boolean insertSpaces) { - int firstNonWhitespaceIndex = TextUtils.firstNonWhitespaceIndex(str); - if (firstNonWhitespaceIndex == -1) { - firstNonWhitespaceIndex = str.length(); - } - return TextUtils.normalizeIndentationFromWhitespace(str.substring(0, firstNonWhitespaceIndex), tabSize, - insertSpaces) + str.substring(firstNonWhitespaceIndex); - } - - - private static String normalizeIndentationFromWhitespace(final String str, final int tabSize, - final boolean insertSpaces) { - int spacesCnt = 0; - for (int i = 0; i < str.length(); i++) { - if (str.charAt(i) == '\t') { - spacesCnt += tabSize; - } else { - spacesCnt++; - } - } - - final var result = new StringBuilder(); - if (!insertSpaces) { - final long tabsCnt = Math.round(Math.floor((double) spacesCnt / tabSize)); - spacesCnt = spacesCnt % tabSize; - for (int i = 0; i < tabsCnt; i++) { - result.append('\t'); - } - } - - for (int i = 0; i < spacesCnt; i++) { - result.append(' '); - } - - return result.toString(); - } - - /** - * Returns first index of the string that is not whitespace. If string is empty - * or contains only whitespaces, returns -1 - */ - public static int firstNonWhitespaceIndex(final String str) { - for (int i = 0, len = str.length(); i < len; i++) { - final char c = str.charAt(i); - if (c != ' ' && c != '\t') { - return i; - } - } - return -1; - } - - public static String getIndentationFromWhitespace(final String whitespace, final TabSpacesInfo tabSpaces) { - final var tab = "\t"; //$NON-NLS-1$ - int indentOffset = 0; - boolean startsWithTab = true; - boolean startsWithSpaces = true; - final String spaces = tabSpaces.isInsertSpaces() - ? " ".repeat(tabSpaces.getTabSize()) - : ""; - while (startsWithTab || startsWithSpaces) { - startsWithTab = whitespace.startsWith(tab, indentOffset); - startsWithSpaces = tabSpaces.isInsertSpaces() && whitespace.startsWith(spaces, indentOffset); - if (startsWithTab) { - indentOffset += tab.length(); - } - if (startsWithSpaces) { - indentOffset += spaces.length(); - } - } - return whitespace.substring(0, indentOffset); - } - - public static String getLinePrefixingWhitespaceAtPosition(final Content d, final CharPosition position) { - - var line = d.getLine(position.line); - - var startIndex = IntPair.getFirst(io.github.rosemoe.sora.text.TextUtils.findLeadingAndTrailingWhitespacePos( - line - )); - - return line.subSequence(0, startIndex).toString(); - - } - - - /** - * Returns the leading whitespace of the string. - * If the string contains only whitespaces, returns entire string - */ - public static String getLeadingWhitespace(String str, int start, int end) { - for (var i = start; i < end; i++) { - var chCode = str.charAt(i); - if (chCode != 32 /*CharCode.Space*/ && chCode != 9/*CharCode.Tab*/) { - return str.substring(start, i); - } - } - return str.substring(start, end); - } - - /** - * Returns the leading whitespace of the string. - * If the string contains only whitespaces, returns entire string - */ - public static String getLeadingWhitespace(String str) { - return getLeadingWhitespace(str, 0, str.length()); - } - - - public static TabSpacesInfo getTabSpaces(final TextMateLanguage language) { - return new TabSpacesInfo(language.getTabSize(), !language.useTab()); - } - - public static String firstCharToUpperCase(String targetString) { - char[] cs = targetString.toCharArray(); - cs[0] -= 32; - return String.valueOf(cs); - } - - public static boolean isEmpty(String text) { - if (text.length()<1) { - return true; - } - for (var i = 0; i < text.length(); i++) { - var chCode = text.charAt(i); - if (chCode != 32 /*CharCode.Space*/ && chCode != 9/*CharCode.Tab*/) { - return false; - } - } - return true; - } -} From 035eb88284478a00f64ea874cad87f879e8aadc8 Mon Sep 17 00:00:00 2001 From: dingyi Date: Mon, 29 Jan 2024 20:43:14 +0800 Subject: [PATCH 2/9] fix(editor): fix #532 --- .../java/io/github/rosemoe/sora/widget/CodeEditor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/editor/src/main/java/io/github/rosemoe/sora/widget/CodeEditor.java b/editor/src/main/java/io/github/rosemoe/sora/widget/CodeEditor.java index 3fca239f9..93d397e45 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/widget/CodeEditor.java +++ b/editor/src/main/java/io/github/rosemoe/sora/widget/CodeEditor.java @@ -524,7 +524,7 @@ protected void initialize(AttributeSet attrs, int defStyleAttr, int defStyleRes) TypedArray a = null; try { a = getContext().obtainStyledAttributes(new int[]{android.R.attr.listPreferredItemHeight}); - verticalScrollFactor = a.getFloat(0, 32); + verticalScrollFactor = a.getDimension(0, 32); } catch (Exception e) { Log.e(LOG_TAG, "Failed to get scroll factor, using default.", e); verticalScrollFactor = 32; @@ -1684,9 +1684,9 @@ public void indentSelection() { /** * Indents the lines. Does nothing if the onlyIfSelected is true and * no text is selected. - * + * * @param onlyIfSelected Set to true if lines must be indented only if the text is - * selected. + * selected. */ public void indentLines(boolean onlyIfSelected) { @@ -1698,7 +1698,7 @@ public void indentLines(boolean onlyIfSelected) { final var tabString = createTabString(); final var text = getText(); final var tabWidth = getTabWidth(); - + text.beginBatchEdit(); for (int i = cursor.getLeftLine(); i <= cursor.getRightLine(); i++) { final var line = text.getLine(i); @@ -4134,9 +4134,9 @@ public void hideSoftInput() { * Check whether the soft keyboard is enabled for this editor. Unlike {@link #isSoftKeyboardEnabled()}, * this method also checks whether a hardware keyboard is connected. * + * @return Whether the editor should show soft keyboard. * @see #isSoftKeyboardEnabled() * @see #isDisableSoftKbdIfHardKbdAvailable() - * @return Whether the editor should show soft keyboard. */ protected boolean checkSoftInputEnabled() { if (isDisableSoftKbdIfHardKbdAvailable() From 5bd5602741d723d89d894b48b47f7cc7e1dbf8dc Mon Sep 17 00:00:00 2001 From: dingyi Date: Mon, 29 Jan 2024 21:48:38 +0800 Subject: [PATCH 3/9] fix(editor): fix #420 --- .../component/DefaultCompletionLayout.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/editor/src/main/java/io/github/rosemoe/sora/widget/component/DefaultCompletionLayout.java b/editor/src/main/java/io/github/rosemoe/sora/widget/component/DefaultCompletionLayout.java index 365e1da72..6b970599b 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/widget/component/DefaultCompletionLayout.java +++ b/editor/src/main/java/io/github/rosemoe/sora/widget/component/DefaultCompletionLayout.java @@ -25,12 +25,14 @@ import android.animation.LayoutTransition; import android.content.Context; +import android.graphics.Outline; import android.graphics.drawable.GradientDrawable; import android.os.SystemClock; import android.util.TypedValue; import android.view.MotionEvent; import android.view.View; import android.view.ViewGroup; +import android.view.ViewOutlineProvider; import android.widget.LinearLayout; import android.widget.ListView; import android.widget.ProgressBar; @@ -115,6 +117,8 @@ public View inflate(@NonNull Context context) { rootLayout.setBackground(gd); + setRootViewOutlineProvider(rootView); + listView.setDividerHeight(0); setLoading(true); @@ -127,6 +131,8 @@ public View inflate(@NonNull Context context) { } }); + + return rootLayout; } @@ -137,6 +143,8 @@ public void onApplyColorScheme(@NonNull EditorColorScheme colorScheme) { gd.setStroke(1, colorScheme.getColor(EditorColorScheme.COMPLETION_WND_CORNER)); gd.setColor(colorScheme.getColor(EditorColorScheme.COMPLETION_WND_BACKGROUND)); rootView.setBackground(gd); + + setRootViewOutlineProvider(rootView); } @Override @@ -170,6 +178,16 @@ private void performScrollList(int offset) { ev.recycle(); } + private void setRootViewOutlineProvider(View rootView) { + rootView.setOutlineProvider(new ViewOutlineProvider() { + @Override + public void getOutline(View view, Outline outline) { + outline.setRoundRect(0, 0, view.getWidth(), view.getHeight(), TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, 8, view.getContext().getResources().getDisplayMetrics())); + } + }); + rootView.setClipToOutline(true); + } + @Override public void ensureListPositionVisible(int position, int increment) { listView.post(() -> { From 87696eae15af7ac7194485646b0bbf4a119b663a Mon Sep 17 00:00:00 2001 From: dingyi Date: Mon, 29 Jan 2024 23:03:33 +0800 Subject: [PATCH 4/9] fix(lang-textmate): enable symbol pair matching by default --- app/src/main/assets/textmate/lua/package.json | 43 ------------------- .../sora/widget/EditorInputConnection.java | 2 +- .../sora/langs/textmate/TextMateLanguage.java | 3 +- .../textmate/TextMateSymbolPairMatch.java | 6 +-- .../registry/FileProviderRegistry.java | 2 +- .../textmate/registry/GrammarRegistry.java | 1 - .../registry/provider/AssetsFileResolver.java | 1 - 7 files changed, 4 insertions(+), 54 deletions(-) delete mode 100644 app/src/main/assets/textmate/lua/package.json diff --git a/app/src/main/assets/textmate/lua/package.json b/app/src/main/assets/textmate/lua/package.json deleted file mode 100644 index dc1228d55..000000000 --- a/app/src/main/assets/textmate/lua/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "lua", - "displayName": "%displayName%", - "description": "%description%", - "version": "1.0.0", - "publisher": "vscode", - "license": "MIT", - "engines": { - "vscode": "*" - }, - "scripts": { - "update-grammar": "node ../node_modules/vscode-grammar-updater/bin sumneko/lua.tmbundle Syntaxes/Lua.plist ./syntaxes/lua.tmLanguage.json" - }, - "contributes": { - "languages": [ - { - "id": "lua", - "extensions": [ - ".lua" - ], - "aliases": [ - "Lua", - "lua" - ], - "configuration": "./language-configuration.json" - } - ], - "grammars": [ - { - "language": "lua", - "scopeName": "source.lua", - "path": "./syntaxes/lua.tmLanguage.json", - "tokenTypes": { - "comment.line.double-dash.doc.lua": "other" - } - } - ] - }, - "repository": { - "type": "git", - "url": "https://github.com/microsoft/vscode.git" - } -} \ No newline at end of file diff --git a/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java b/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java index 2f0ccc5b5..f6b53838c 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java +++ b/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java @@ -332,7 +332,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI editor.setSelectionRegion(editorCursor.getLeftLine(), editorCursor.getLeftColumn(), editorCursor.getRightLine(), editorCursor.getRightColumn() - pair.close.length()); } else if (editorCursor.isSelected() && editor.getEditorLanguage().getQuickQuoteHandler() != null) { - editor.commitText(text,applyAutoIndent); + editor.commitText(text, applyAutoIndent); } else { editorText.beginBatchEdit(); diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java index 1b65ac7ba..48d5df178 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateLanguage.java @@ -94,7 +94,6 @@ protected TextMateLanguage(IGrammar grammar, symbolPairMatch = new TextMateSymbolPairMatch(this); createAnalyzerAndNewlineHandler(grammar, languageConfiguration); - } @@ -272,7 +271,7 @@ public TextMateNewlineHandler getNewlineHandler() { } @Override - public SymbolPairMatch getSymbolPairs() { + public TextMateSymbolPairMatch getSymbolPairs() { return symbolPairMatch; } diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java index 347beac59..3404f8b85 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java @@ -45,7 +45,7 @@ public class TextMateSymbolPairMatch extends SymbolPairMatch { private final TextMateLanguage language; - private boolean enabled; + private boolean enabled = true; public TextMateSymbolPairMatch(TextMateLanguage language) { super(new SymbolPairMatch.DefaultSymbolPairs()); @@ -190,11 +190,7 @@ public boolean shouldDoReplace(CodeEditor editor, ContentLine contentLine, int l var spansOnCurrentLine = editor.getSpansForLine(currentLine); var currentSpan = binarySearchSpan(spansOnCurrentLine, currentColumn); - - var extra = currentSpan.extra; - - if (extra instanceof Integer) { var index = Arrays.binarySearch(notInTokenTypeArray, (Integer) extra); return index < 0; diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/FileProviderRegistry.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/FileProviderRegistry.java index 5fca688fc..b49349215 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/FileProviderRegistry.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/FileProviderRegistry.java @@ -64,7 +64,7 @@ public synchronized void removeFileProvider(FileResolver fileResolver) { public InputStream tryGetInputStream(String path) { for (var provider : allFileResolvers) { var stream = provider.resolveStreamByPath(path); - if (stream!=null) { + if (stream != null) { return stream; } } diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java index e1e99063b..e80f72eb2 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/GrammarRegistry.java @@ -210,7 +210,6 @@ private synchronized IGrammar doLoadGrammar(GrammarDefinition grammarDefinition) if (languageConfigurationStream != null) { - var languageConfiguration = LanguageConfiguration.load( new InputStreamReader(languageConfigurationStream) ); diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/provider/AssetsFileResolver.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/provider/AssetsFileResolver.java index becfe6356..a5cf9c2e4 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/provider/AssetsFileResolver.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/registry/provider/AssetsFileResolver.java @@ -41,7 +41,6 @@ public AssetsFileResolver(AssetManager assetManager) { @Nullable @Override public InputStream resolveStreamByPath(String path) { - try { return assetManager.open(path); } catch (IOException e) { From d24c5d7c852ecd35c0dd12f1fe79e9c783fb8766 Mon Sep 17 00:00:00 2001 From: dingyi Date: Mon, 29 Jan 2024 23:39:40 +0800 Subject: [PATCH 5/9] fix(lang-textmate): make auto surrounding work properly --- .../sora/widget/EditorInputConnection.java | 2 ++ .../textmate/TextMateSymbolPairMatch.java | 8 ++------ .../model/AutoClosingPairConditional.java | 3 +++ .../internal/model/CharacterPair.java | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java b/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java index f6b53838c..f074b1f2a 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java +++ b/editor/src/main/java/io/github/rosemoe/sora/widget/EditorInputConnection.java @@ -286,6 +286,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI } else if (composingText.isComposing()) { deleteComposingText(); } + // replace text SymbolPairMatch.SymbolPair pair = null; if (editor.getProps().symbolPairAutoCompletion && text.length() > 0) { @@ -303,6 +304,7 @@ protected void commitTextInternal(@NonNull CharSequence text, boolean applyAutoI inputText, firstCharFromText ); } + var editorText = editor.getText(); var editorCursor = editor.getCursor(); diff --git a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java index 3404f8b85..f88b3e6a6 100644 --- a/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java +++ b/language-textmate/src/main/java/io/github/rosemoe/sora/langs/textmate/TextMateSymbolPairMatch.java @@ -78,7 +78,6 @@ public void updatePair() { removeAllPairs(); - var surroundingPairs = languageConfiguration.getSurroundingPairs(); var autoClosingPairs = languageConfiguration.getAutoClosingPairs(); @@ -89,7 +88,6 @@ public void updatePair() { mergePairs.addAll(autoClosingPairs); } - if (surroundingPairs != null) { for (var surroundingPair : surroundingPairs) { @@ -111,7 +109,6 @@ public void updatePair() { } mergePairs.add(newPair); - } } @@ -168,14 +165,12 @@ public SymbolPairEx(AutoClosingPairConditional pair) { } Arrays.sort(notInTokenTypeArray); - } @Override public boolean shouldDoReplace(CodeEditor editor, ContentLine contentLine, int leftColumn) { - if (editor.getCursor().isSelected()) { - return true; + return isSurroundingPair; } if (notInTokenTypeArray == null) { @@ -191,6 +186,7 @@ public boolean shouldDoReplace(CodeEditor editor, ContentLine contentLine, int l var currentSpan = binarySearchSpan(spansOnCurrentLine, currentColumn); var extra = currentSpan.extra; + if (extra instanceof Integer) { var index = Arrays.binarySearch(notInTokenTypeArray, (Integer) extra); return index < 0; diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java index 1ccef74ff..8e17a4779 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/AutoClosingPairConditional.java @@ -11,6 +11,8 @@ */ package org.eclipse.tm4e.languageconfiguration.internal.model; +import androidx.annotation.NonNull; + import java.util.List; import org.eclipse.tm4e.core.internal.utils.StringUtils; @@ -29,6 +31,7 @@ public AutoClosingPairConditional(final String open, final String close, final L this.notIn = notIn; } + @NonNull @Override public String toString() { return StringUtils.toString(this, sb -> sb diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java index 8c4efe8c3..0d9d049ff 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/languageconfiguration/internal/model/CharacterPair.java @@ -11,8 +11,12 @@ */ package org.eclipse.tm4e.languageconfiguration.internal.model; +import androidx.annotation.NonNull; + import org.eclipse.tm4e.core.internal.utils.StringUtils; +import java.util.Objects; + /** * A tuple of two characters, like a pair of opening and closing brackets. * @@ -30,6 +34,20 @@ public CharacterPair(final String opening, final String closing) { this.close = closing; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CharacterPair that = (CharacterPair) o; + return Objects.equals(open, that.open) && Objects.equals(close, that.close); + } + + @Override + public int hashCode() { + return Objects.hash(open, close); + } + + @NonNull @Override public String toString() { return StringUtils.toString(this, sb -> sb From 50d8cc85f867cec2bceecc2aa6e1d519020732c3 Mon Sep 17 00:00:00 2001 From: dingyi Date: Tue, 30 Jan 2024 01:28:04 +0800 Subject: [PATCH 6/9] fix(editor): change the completion item order from ascending to descending We Use fuzzyScore to generate scores between completions and input text, and sort them accordingly. Previously, the scores were sorted in ascending order, which was incorrect since higher scores should be closer to the input text. I apologize for taking so long to realize this mistake. --- .../lang/completion/CompletionPublisher.java | 4 +- .../sora/lang/completion/comparators.kt | 94 +++++++++++++------ 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/CompletionPublisher.java b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/CompletionPublisher.java index 2fae52fff..8ba3ed973 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/CompletionPublisher.java +++ b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/CompletionPublisher.java @@ -123,7 +123,7 @@ public void setComparator(@Nullable Comparator comparator) { return; } this.comparator = comparator; - if (items.size() != 0 && comparator != null) { + if (!items.isEmpty() && comparator != null) { handler.post(() -> { if (invalid) { return; @@ -217,7 +217,7 @@ public void updateList(boolean forced) { if (locked) { try { - if (candidates.size() == 0) { + if (candidates.isEmpty()) { callback.run(); return; } diff --git a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt index 339e7e9b2..b939df303 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt +++ b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt @@ -30,21 +30,21 @@ import io.github.rosemoe.sora.text.CharPosition import io.github.rosemoe.sora.text.ContentReference import io.github.rosemoe.sora.util.CharCode -fun fuzzyComparator(a: SortedCompletionItem, b: SortedCompletionItem): Int { - return if (a.score.score > b.score.score) { - -1; - } else if (a.score.score < b.score.score) { - 1 - } else { - snippetUpComparator(a.completionItem, b.completionItem) - } -} - private fun CharSequence?.asString(): String { return if (this == null) " " else if (this is String) this else this.toString() } fun defaultComparator(a: CompletionItem, b: CompletionItem): Int { + // check score + val p1Score = (a.extra as SortedCompletionItem?)?.score?.score ?: 0 + val p2Score = (b.extra as SortedCompletionItem?)?.score?.score ?: 0 + + // if score biggest, it better similar to input text + if (p1Score < p2Score) { + return 1; + } else if (p1Score > p2Score) { + return -1; + } var p1 = a.sortText.asString() var p2 = b.sortText.asString() @@ -67,25 +67,19 @@ fun defaultComparator(a: CompletionItem, b: CompletionItem): Int { return 1; } - // check with 'kind' + // if kind biggest, it better important + val kind = (b.kind?.value ?: 0) - (a.kind?.value ?: 0) - val kind = (a.kind?.value ?: 0) - (b.kind?.value ?: 0) - - return if (kind == 0) { - if (a.extra is SortedCompletionItem && b.extra is SortedCompletionItem) { - return fuzzyComparator(a.extra as SortedCompletionItem, b.extra as SortedCompletionItem) - } else kind - } else kind - + return kind } fun snippetUpComparator(a: CompletionItem, b: CompletionItem): Int { if (a.kind != b.kind) { if (a.kind == CompletionItemKind.Snippet) { - return -1; - } else if (b.kind == CompletionItemKind.Snippet) { return 1; + } else if (b.kind == CompletionItemKind.Snippet) { + return -1; } } return defaultComparator(a, b); @@ -100,7 +94,7 @@ fun getCompletionItemComparator( source.validateAccess() - val sourceLine = source.reference.getLine(cursorPosition.line); + val sourceLine = source.reference.getLine(cursorPosition.line) var word = "" var wordLow = "" @@ -153,8 +147,14 @@ fun getCompletionItemComparator( // the fallback-sort using the initial sort order. // use a score of `-100` because that is out of the // bound of values `fuzzyScore` will return - - if (wordLen > 0) { + if (wordLen == 0) { + // when there is nothing to score against, don't + // event try to do. Use a const rank and rely on + // the fallback-sort using the initial sort order. + // use a score of `-100` because that is out of the + // bound of values `fuzzyScore` will return + item.score = FuzzyScore.default + } else { // skip word characters that are whitespace until // we have hit the replace range (overwriteBefore) var wordPos = 0; @@ -170,7 +170,42 @@ fun getCompletionItemComparator( if (wordPos >= wordLen) { // the wordPos at which scoring starts is the whole word // and therefore the same rules as not having a word apply - item.score = FuzzyScore.default + item.score = FuzzyScore.default; + } else if (originItem.sortText?.isNotEmpty() == true) { + // when there is a `filterText` it must match the `word`. + // if it matches we check with the label to compute highlights + // and if that doesn't yield a result we have no highlights, + // despite having the match + // by default match `word` against the `label` + val match = scoreFn.calculateScore( + word, + wordLow, + wordPos, + originItem.sortText.asString(), + originItem.sortText.asString().lowercase(), + 0, + FuzzyScoreOptions.default + ) ?: continue; // NO match + + // compareIgnoreCase(item.completion.filterText, item.textLabel) === 0 + if (originItem.sortText === originItem.label) { + // filterText and label are actually the same -> use good highlights + item.score = match; + } else { + // re-run the scorer on the label in the hope of a result BUT use the rank + // of the filterText-match + val labelMatch = scoreFn.calculateScore( + word, + wordLow, + wordPos, + originItem.label.asString(), + originItem.label.asString().lowercase(), + 0, + FuzzyScoreOptions.default + ) ?: continue; // NO match + item.score = labelMatch + labelMatch.matches[0] = match.matches[0] + } } else { // by default match `word` against the `label` @@ -185,12 +220,11 @@ fun getCompletionItemComparator( ) ?: continue; // NO match item.score = match; } - } - - originItem.extra = item - } + originItem.extra = item + } + } return Comparator { o1, o2 -> snippetUpComparator(o1, o2) @@ -200,4 +234,4 @@ fun getCompletionItemComparator( data class SortedCompletionItem( val completionItem: CompletionItem, var score: FuzzyScore -) \ No newline at end of file +) From f53851a5c0fc658f71ee906be812e71f811c0ae1 Mon Sep 17 00:00:00 2001 From: dingyi Date: Tue, 30 Jan 2024 02:01:23 +0800 Subject: [PATCH 7/9] refactor(editor): do not force cast `CompletionItem#extra` to `SortedCompletionItem` --- .../io/github/rosemoe/sora/lang/completion/comparators.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt index b939df303..ee346dac3 100644 --- a/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt +++ b/editor/src/main/java/io/github/rosemoe/sora/lang/completion/comparators.kt @@ -36,8 +36,8 @@ private fun CharSequence?.asString(): String { fun defaultComparator(a: CompletionItem, b: CompletionItem): Int { // check score - val p1Score = (a.extra as SortedCompletionItem?)?.score?.score ?: 0 - val p2Score = (b.extra as SortedCompletionItem?)?.score?.score ?: 0 + val p1Score = (a.extra as? SortedCompletionItem)?.score?.score ?: 0 + val p2Score = (b.extra as? SortedCompletionItem)?.score?.score ?: 0 // if score biggest, it better similar to input text if (p1Score < p2Score) { From db2bb59068befa91363441631ae0616eaf5630b9 Mon Sep 17 00:00:00 2001 From: dingyi Date: Tue, 30 Jan 2024 04:43:13 +0800 Subject: [PATCH 8/9] feat(lang-textmate): inappropriate cache saving & restoring in `OnigRegExp` --- .../core/internal/oniguruma/OnigRegExp.java | 172 +++++++++--------- 1 file changed, 88 insertions(+), 84 deletions(-) diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java index 76b02c26c..57b7b89be 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java @@ -3,13 +3,13 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - * + *

* SPDX-License-Identifier: EPL-2.0 - * + *

* Initial code from https://github.com/atom/node-oniguruma * Initial copyright Copyright (c) 2013 GitHub Inc. * Initial license: MIT - * + *

* Contributors: * - GitHub Inc.: Initial code, written in JavaScript, licensed under MIT license * - Angelo Zerr - translation and adaptation to Java @@ -42,85 +42,89 @@ public final class OnigRegExp { @Nullable - private OnigString lastSearchString; - - private int lastSearchPosition = -1; - - @Nullable - private OnigResult lastSearchResult; - - private final String pattern; - private final Regex regex; - - private final boolean hasGAnchor; - - /** - * @throws TMException if parsing fails - */ - public OnigRegExp(final String pattern) { - this(pattern, false); - } - - /** - * @throws TMException if parsing fails - */ - public OnigRegExp(final String pattern, final boolean ignoreCase) { - this.pattern = pattern; - hasGAnchor = pattern.contains("\\G"); - final byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); - try { - int options = Option.CAPTURE_GROUP; - if (ignoreCase) - options |= Option.IGNORECASE; - regex = new Regex(patternBytes, 0, patternBytes.length, options, UTF8Encoding.INSTANCE, Syntax.DEFAULT, - /*LOGGER.isLoggable(Level.WARNING) ? LOGGER_WARN_CALLBACK :*/ WarnCallback.NONE); - } catch (final SyntaxException ex) { - throw new TMException("Parsing regex pattern \"" + pattern + "\" failed with " + ex, ex); - } - } - - /** - * @return null if not found - */ - public @Nullable OnigResult search(final OnigString str, final int startPosition) { - if (hasGAnchor) { - // Should not use caching, because the regular expression - // targets the current search position (\G) - return search(str.bytesUTF8, startPosition, str.bytesCount); - } - - final var lastSearchResult0 = this.lastSearchResult; - if (lastSearchString == str - && lastSearchPosition <= startPosition - && (lastSearchResult0 == null || lastSearchResult0.locationAt(0) >= startPosition)) { - return lastSearchResult0; - } - - lastSearchString = str; - lastSearchPosition = startPosition; - lastSearchResult = search(str.bytesUTF8, startPosition, str.bytesCount); - return lastSearchResult; - } - - @Nullable - private OnigResult search(final byte[] data, final int startPosition, final int end) { - final Matcher matcher = regex.matcher(data); - final int status = matcher.search(startPosition, end, Option.DEFAULT); - if (status != Matcher.FAILED) { - final Region region = matcher.getEagerRegion(); - return new OnigResult(region, -1); - } - return null; - } - - public String pattern() { - return pattern; - } - - @Override - public String toString() { - return StringUtils.toString(this, sb -> { - sb.append("pattern=").append(pattern); - }); - } + private OnigString lastSearchString; + + private int lastSearchPosition = -1; + + @Nullable + private OnigResult lastSearchResult; + + private final String pattern; + private final Regex regex; + + private final boolean hasGAnchor; + + /** + * @throws TMException if parsing fails + */ + public OnigRegExp(final String pattern) { + this(pattern, false); + } + + /** + * @throws TMException if parsing fails + */ + public OnigRegExp(final String pattern, final boolean ignoreCase) { + this.pattern = pattern; + hasGAnchor = pattern.contains("\\G"); + final byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + int options = Option.CAPTURE_GROUP; + if (ignoreCase) + options |= Option.IGNORECASE; + regex = new Regex(patternBytes, 0, patternBytes.length, options, UTF8Encoding.INSTANCE, Syntax.DEFAULT, + /*LOGGER.isLoggable(Level.WARNING) ? LOGGER_WARN_CALLBACK :*/ WarnCallback.NONE); + } catch (final SyntaxException ex) { + throw new TMException("Parsing regex pattern \"" + pattern + "\" failed with " + ex, ex); + } + } + + /** + * @return null if not found + */ + public @Nullable OnigResult search(final OnigString str, final int startPosition) { + if (hasGAnchor) { + // Should not use caching, because the regular expression + // targets the current search position (\G) + return search(str.bytesUTF8, startPosition, str.bytesCount); + } + + synchronized (this) { + final var lastSearchResult0 = this.lastSearchResult; + if (lastSearchString == str + && lastSearchPosition <= startPosition + && (lastSearchResult0 == null || lastSearchResult0.locationAt(0) >= startPosition)) { + return lastSearchResult0; + } + } + + synchronized (this) { + lastSearchString = str; + lastSearchPosition = startPosition; + lastSearchResult = search(str.bytesUTF8, startPosition, str.bytesCount); + } + return lastSearchResult; + } + + @Nullable + private OnigResult search(final byte[] data, final int startPosition, final int end) { + final Matcher matcher = regex.matcher(data); + final int status = matcher.search(startPosition, end, Option.DEFAULT); + if (status != Matcher.FAILED) { + final Region region = matcher.getEagerRegion(); + return new OnigResult(region, -1); + } + return null; + } + + public String pattern() { + return pattern; + } + + @Override + public String toString() { + return StringUtils.toString(this, sb -> { + sb.append("pattern=").append(pattern); + }); + } } From e751bd31f722432382fb6da37da9469026dca78f Mon Sep 17 00:00:00 2001 From: dingyi Date: Tue, 30 Jan 2024 11:27:00 +0800 Subject: [PATCH 9/9] perf(lang-textmate): call caching in `OnigRegExp` --- .../org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java index 57b7b89be..7a2704e27 100644 --- a/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java +++ b/language-textmate/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java @@ -98,10 +98,11 @@ public OnigRegExp(final String pattern, final boolean ignoreCase) { } } + var result = search(str.bytesUTF8, startPosition, str.bytesCount); synchronized (this) { lastSearchString = str; lastSearchPosition = startPosition; - lastSearchResult = search(str.bytesUTF8, startPosition, str.bytesCount); + lastSearchResult = result; } return lastSearchResult; }