feat: add option to display TextMate token info in hover

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java

@@ -20,6 +20,7 @@
 import java.lang.System.Logger;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.BlockingQueue;
@@ -255,7 +256,7 @@ private void revalidateTokens() {
 				// check if complete line was tokenized
 				if (r.stoppedEarly) {
 					// treat the rest of the line as one default token
-					r.tokens.add(new TMToken(r.actualStopOffset, ""));
+					r.tokens.add(new TMToken(r.actualStopOffset, "", Collections.emptyList()));
 					// Use the line's starting state as end state in case of incomplete tokenization
 					r.endState = currLineTokens.startState;
 				}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java

@@ -16,6 +16,8 @@
  */
 package org.eclipse.tm4e.core.model;
 
+import java.util.List;
+
 import org.eclipse.jdt.annotation.Nullable;
 
 /**
@@ -28,10 +30,12 @@ public final class TMToken {
 	public final int startIndex;
 	public final String type;
 	// public readonly language: string
+	public final List<String> scopes;
 
-	public TMToken(final int startIndex, final String type) {
+	public TMToken(final int startIndex, final String type, final List<String> scopes) {
 		this.startIndex = startIndex;
 		this.type = type;
+		this.scopes = scopes;
 	}
 
 	@Override

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java

@@ -29,6 +29,7 @@
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.grammar.IGrammar;
 import org.eclipse.tm4e.core.grammar.IStateStack;
+import org.eclipse.tm4e.core.grammar.IToken;
 import org.eclipse.tm4e.core.internal.grammar.StateStack;
 import org.eclipse.tm4e.core.internal.utils.MoreCollections;
 import org.eclipse.tm4e.core.internal.utils.StringUtils;
@@ -83,13 +84,12 @@ public TokenizationResult tokenize(final String line,
 		// Create the result early and fill in the tokens later
 		final var tmTokens = new ArrayList<TMToken>(tokens.length < 10 ? tokens.length : 10);
 		String lastTokenType = null;
-		for (final var token : tokens) {
+		for (final IToken token : tokens) {
 			final String tokenType = decodeTextMateTokenCached.apply(decodeMap, token.getScopes());
 
 			// do not push a new token if the type is exactly the same (also helps with ligatures)
 			if (!tokenType.equals(lastTokenType)) {
-				final int tokenStartIndex = token.getStartIndex();
-				tmTokens.add(new TMToken(tokenStartIndex + offsetDelta, tokenType));
+				tmTokens.add(new TMToken(token.getStartIndex() + offsetDelta, tokenType, token.getScopes()));
 				lastTokenType = tokenType;
 			}
 		}

org.eclipse.tm4e.ui/plugin.xml

@@ -37,7 +37,7 @@
              path="./themes/Eclipse-light.css" />
       <theme id="org.eclipse.tm4e.ui.themes.WtpXmlClassic"
              name="%Theme.WtpXmlClassic.name"
-             path="./themes/WTP-XML-Classic.css" />  
+             path="./themes/WTP-XML-Classic.css" />
       <!-- "Dark" themes -->
       <theme id="org.eclipse.tm4e.ui.themes.Dark"
              name="%Theme.Dark.name"
@@ -138,4 +138,10 @@
       <super type="org.eclipse.tm4e.ui.textmarker"/>
       <persistent value="true"/>
    </extension>
+
+   <extension point="org.eclipse.ui.genericeditor.hoverProviders">
+      <hoverProvider
+            class="org.eclipse.tm4e.ui.internal.hover.TMTokenTextHover"
+            contentType="org.eclipse.core.runtime.text" />
+   </extension>
 </plugin>

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/TMUIPlugin.java

@@ -43,6 +43,10 @@ public class TMUIPlugin extends AbstractUIPlugin {
 	@Nullable
 	private static volatile TMUIPlugin plugin;
 
+	public static boolean getPreference(final String key, final boolean defaultValue) {
+		return Platform.getPreferencesService().getBoolean(PLUGIN_ID, key, defaultValue, null /* = search in all available scopes */);
+	}
+
 	public static @Nullable String getPreference(final String key, final @Nullable String defaultValue) {
 		return Platform.getPreferencesService().getString(PLUGIN_ID, key, defaultValue, null /* = search in all available scopes */);
 	}

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/TMUIMessages.java

@@ -34,6 +34,7 @@ public final class TMUIMessages extends NLS {
 	public static String TextMatePreferencePage_LanguageConfigurationRelatedLink;
 	public static String TextMatePreferencePage_TaskTagsRelatedLink;
 	public static String TextMatePreferencePage_ThemeRelatedLink;
+	public static String TextMatePreferencePage_ShowTextMateTokenInfoHover;
 
 	// Grammar preferences page
 	public static String GrammarPreferencePage_title;

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/TMUIMessages.properties

@@ -21,6 +21,7 @@ TextMatePreferencePage_GrammarRelatedLink=See <a>''{0}''</a> for associating edi
 TextMatePreferencePage_LanguageConfigurationRelatedLink=See <a>''{0}''</a> for associating editors with language configurations.
 TextMatePreferencePage_TaskTagsRelatedLink=See <a>''{0}''</a> for task tags configuration.
 TextMatePreferencePage_ThemeRelatedLink=See <a>''{0}''</a> for associating editors with themes.
+TextMatePreferencePage_ShowTextMateTokenInfoHover=Show TextMate token info in hovers.
 
 GrammarPreferencePage_title=TextMate grammars
 GrammarPreferencePage_description=Register, configure or remove TextMate grammars:

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java

@@ -0,0 +1,116 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Vegard IT GmbH and others.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Sebastian Thomschke (Vegard IT) - initial implementation
+ *******************************************************************************/
+package org.eclipse.tm4e.ui.internal.hover;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.eclipse.jdt.annotation.Nullable;
+import org.eclipse.jface.text.AbstractReusableInformationControlCreator;
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.DefaultInformationControl;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.IInformationControl;
+import org.eclipse.jface.text.IInformationControlCreator;
+import org.eclipse.jface.text.IRegion;
+import org.eclipse.jface.text.ITextHover;
+import org.eclipse.jface.text.ITextHoverExtension;
+import org.eclipse.jface.text.ITextViewer;
+import org.eclipse.jface.text.Region;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.tm4e.core.model.TMToken;
+import org.eclipse.tm4e.ui.internal.model.TMDocumentModel;
+import org.eclipse.tm4e.ui.internal.model.TMModelManager;
+import org.eclipse.tm4e.ui.internal.preferences.PreferenceHelper;
+import org.eclipse.ui.editors.text.EditorsUI;
+
+public class TMTokenTextHover implements ITextHover, ITextHoverExtension {
+
+	private static final class RegionWithTMToken extends Region {
+		final TMToken token;
+		final String tokenText;
+
+		RegionWithTMToken(final int offset, final int length, final String tokenText, final TMToken token) {
+			super(offset, length);
+			this.tokenText = tokenText;
+			this.token = token;
+		}
+	}
+
+	@Override
+	public IInformationControlCreator getHoverControlCreator() {
+		// setup a hover control that interprets basic HTML input
+		return new AbstractReusableInformationControlCreator() {
+			@Override
+			protected IInformationControl doCreateInformationControl(final @NonNullByDefault({}) Shell parent) {
+				return new DefaultInformationControl(parent, EditorsUI.getTooltipAffordanceString());
+			}
+		};
+	}
+
+	@Override
+	public @Nullable String getHoverInfo(final @NonNullByDefault({}) ITextViewer textViewer,
+			final @NonNullByDefault({}) IRegion hoverRegion) {
+		if (hoverRegion instanceof final RegionWithTMToken regionWithToken) {
+			final var text = regionWithToken.tokenText.replace(' ', '·').replace('\t', '→');
+			return "<b>" + text + "</b> (" + text.length()
+					+ " chars)<br>"
+					+ "<br>"
+					+ "<b>Token Type:</b> " + regionWithToken.token.type + "<br>"
+					+ "<b>TextMate Scopes:</b> <li>" + String.join("<li>", regionWithToken.token.scopes);
+		}
+		return null;
+	}
+
+	@Override
+	public @Nullable IRegion getHoverRegion(final @NonNullByDefault({}) ITextViewer textViewer, final int offset) {
+		if (!PreferenceHelper.isTMTokenHoverEnabled())
+			return null;
+
+		final @Nullable IDocument doc = textViewer.getDocument();
+		if (doc == null)
+			return null;
+
+		final TMDocumentModel model = TMModelManager.INSTANCE.getConnectedModel(doc);
+		if (model == null)
+			return null;
+
+		try {
+			// retrieve parsed TM tokens of the hovered line
+			final int lineIndex = doc.getLineOfOffset(offset);
+			final var tokens = model.getLineTokens(lineIndex);
+			if (tokens == null)
+				return null;
+
+			// find the TM token at the hover position
+			final int lineStartOffset = doc.getLineOffset(lineIndex);
+			TMToken hoveredToken = null;
+			TMToken nextToken = null;
+			for (final TMToken token : tokens) {
+				if (token.startIndex <= offset - lineStartOffset) {
+					hoveredToken = token;
+				} else {
+					nextToken = token;
+					break;
+				}
+			}
+			if (hoveredToken == null)
+				return null;
+
+			final int regionOffset = lineStartOffset + hoveredToken.startIndex;
+			final int regionLength = nextToken == null
+					? doc.getLineLength(lineIndex) - hoveredToken.startIndex
+					: nextToken.startIndex - hoveredToken.startIndex;
+			return new RegionWithTMToken(regionOffset, regionLength, doc.get(regionOffset, regionLength), hoveredToken);
+		} catch (final BadLocationException e) {
+			return null;
+		}
+	}
+}

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/package-info.java

@@ -0,0 +1,4 @@
+@NonNullByDefault
+package org.eclipse.tm4e.ui.internal.hover;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/model/TMModelManager.java

@@ -16,6 +16,7 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.jface.text.IDocument;
 import org.eclipse.tm4e.ui.model.ITMModelManager;
 
@@ -44,6 +45,10 @@ public void disconnect(final IDocument document) {
 		}
 	}
 
+	public @Nullable TMDocumentModel getConnectedModel(final IDocument document) {
+		return models.get(document);
+	}
+
 	@Override
 	public boolean isConnected(final IDocument document) {
 		return models.containsKey(document);

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/preferences/AbstractPreferencePage.java

@@ -22,7 +22,7 @@ public abstract class AbstractPreferencePage extends PreferencePage implements I
 
 	private final @Nullable String title;
 
-	protected AbstractPreferencePage(final @Nullable String title, final String description) {
+	protected AbstractPreferencePage(final @Nullable String title, final @Nullable String description) {
 		this.title = title;
 		setDescription(description);
 	}

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/preferences/PreferenceConstants.java

@@ -25,6 +25,8 @@ public final class PreferenceConstants {
 	public static final String DEFAULT_DARK_THEME = "org.eclipse.tm4e.ui.themes.defaultDarkTheme";
 	public static final String DEFAULT_LIGHT_THEME = "org.eclipse.tm4e.ui.themes.defaultLightTheme";
 
+	public static final String TMTOKEN_HOVER_ENABLED = "org.eclipse.tm4e.ui.tmScopeHoverEnabled";
+
 	private PreferenceConstants() {
 	}
 }

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/preferences/PreferenceHelper.java

@@ -110,6 +110,16 @@ public static void saveMarkerConfigs(final Set<MarkerConfig> markerConfigs) thro
 		prefs.flush();
 	}
 
+	public static boolean isTMTokenHoverEnabled() {
+		return TMUIPlugin.getPreference(PreferenceConstants.TMTOKEN_HOVER_ENABLED, false);
+	}
+
+	public static void saveTMTokenHoverEnabled(boolean isEnabled) throws BackingStoreException {
+		final var prefs = InstanceScope.INSTANCE.getNode(TMUIPlugin.PLUGIN_ID);
+		prefs.putBoolean(PreferenceConstants.TMTOKEN_HOVER_ENABLED, isEnabled);
+		prefs.flush();
+	}
+
 	private PreferenceHelper() {
 	}
 }