diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java index 88012622..d365f27f 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java @@ -20,6 +20,7 @@ import java.lang.System.Logger; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.concurrent.BlockingQueue; @@ -255,7 +256,7 @@ private void revalidateTokens() { // check if complete line was tokenized if (r.stoppedEarly) { // treat the rest of the line as one default token - r.tokens.add(new TMToken(r.actualStopOffset, "")); + r.tokens.add(new TMToken(r.actualStopOffset, "", Collections.emptyList())); // Use the line's starting state as end state in case of incomplete tokenization r.endState = currLineTokens.startState; } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java index 776ba8de..fda5151f 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java @@ -16,6 +16,8 @@ */ package org.eclipse.tm4e.core.model; +import java.util.List; + import org.eclipse.jdt.annotation.Nullable; /** @@ -28,10 +30,12 @@ public final class TMToken { public final int startIndex; public final String type; // public readonly language: string + public final List scopes; - public TMToken(final int startIndex, final String type) { + public TMToken(final int startIndex, final String type, final List scopes) { this.startIndex = startIndex; this.type = type; + this.scopes = scopes; } @Override diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java index cad68611..bbea2950 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java @@ -29,6 +29,7 @@ import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.grammar.IGrammar; import org.eclipse.tm4e.core.grammar.IStateStack; +import org.eclipse.tm4e.core.grammar.IToken; import org.eclipse.tm4e.core.internal.grammar.StateStack; import org.eclipse.tm4e.core.internal.utils.MoreCollections; import org.eclipse.tm4e.core.internal.utils.StringUtils; @@ -83,13 +84,12 @@ public TokenizationResult tokenize(final String line, // Create the result early and fill in the tokens later final var tmTokens = new ArrayList(tokens.length < 10 ? tokens.length : 10); String lastTokenType = null; - for (final var token : tokens) { + for (final IToken token : tokens) { final String tokenType = decodeTextMateTokenCached.apply(decodeMap, token.getScopes()); // do not push a new token if the type is exactly the same (also helps with ligatures) if (!tokenType.equals(lastTokenType)) { - final int tokenStartIndex = token.getStartIndex(); - tmTokens.add(new TMToken(tokenStartIndex + offsetDelta, tokenType)); + tmTokens.add(new TMToken(token.getStartIndex() + offsetDelta, tokenType, token.getScopes())); lastTokenType = tokenType; } } diff --git a/org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java b/org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java index 8b141b8f..11a0762e 100644 --- a/org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java +++ b/org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java @@ -35,9 +35,11 @@ public class TMTokenTextHover implements ITextHover, ITextHoverExtension { private static final class RegionWithTMToken extends Region { final TMToken token; + final String text; - RegionWithTMToken(final int offset, final int length, final TMToken token) { + RegionWithTMToken(final int offset, final int length, final String text, final TMToken token) { super(offset, length); + this.text = text; this.token = token; } } @@ -57,7 +59,12 @@ protected IInformationControl doCreateInformationControl(final @NonNullByDefault public @Nullable String getHoverInfo(final @NonNullByDefault({}) ITextViewer textViewer, final @NonNullByDefault({}) IRegion hoverRegion) { if (hoverRegion instanceof final RegionWithTMToken regionWithToken) { - return "TextMate scope: " + regionWithToken.token.type; + final var text = regionWithToken.text.replace(' ', '·').replace('\t', '→'); + return "" + text + " (" + text.length() + + " chars)
" + + "
" + + "Token Type: " + regionWithToken.token.type + "
" + + "TextMate Scopes:
  • " + String.join("
  • ", regionWithToken.token.scopes); } return null; } @@ -97,12 +104,11 @@ protected IInformationControl doCreateInformationControl(final @NonNullByDefault if (hoveredToken == null) return null; - return new RegionWithTMToken( - lineStartOffset + hoveredToken.startIndex, - nextToken == null - ? doc.getLineLength(lineIndex) - hoveredToken.startIndex - : nextToken.startIndex - hoveredToken.startIndex, - hoveredToken); + final int regionOffset = lineStartOffset + hoveredToken.startIndex; + final int regionLength = nextToken == null + ? doc.getLineLength(lineIndex) - hoveredToken.startIndex + : nextToken.startIndex - hoveredToken.startIndex; + return new RegionWithTMToken(regionOffset, regionLength, doc.get(regionOffset, regionLength), hoveredToken); } catch (final BadLocationException e) { return null; }