Semantic tokens for typescript

package.json

@@ -516,6 +516,11 @@
           "default": true,
           "description": "Whether to automatic updating import path when rename or move a file"
         },
+        "vetur.languageFeatures.semanticTokens": {
+          "type": "boolean",
+          "default": true,
+          "description": "Whether to enable semantic highlighting. Currently only works for typescript"
+        },
         "vetur.trace.server": {
           "type": "string",
           "enum": [
@@ -555,7 +560,14 @@
           "description": "Enable template interpolation service that offers hover / definition / references in Vue interpolations."
         }
       }
-    }
+    },
+    "semanticTokenScopes": [
+      {
+        "scopes": {
+          "property.refValue": ["entity.name.function"]
+        }
+      }
+    ]
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",

server/src/config.ts

@@ -65,6 +65,7 @@ export interface VLSConfig {
     languageFeatures: {
       codeActions: boolean;
       updateImportOnFileMove: boolean;
+      semanticTokens: boolean;
     };
     trace: {
       server: 'off' | 'messages' | 'verbose';
@@ -128,7 +129,8 @@ export function getDefaultVLSConfig(): VLSFullConfig {
       },
       languageFeatures: {
         codeActions: true,
-        updateImportOnFileMove: true
+        updateImportOnFileMove: true,
+        semanticTokens: true
       },
       trace: {
         server: 'off'

server/src/embeddedSupport/languageModes.ts

@@ -31,7 +31,7 @@ import { getCSSMode, getSCSSMode, getLESSMode, getPostCSSMode } from '../modes/s
 import { getJavascriptMode } from '../modes/script/javascript';
 import { VueHTMLMode } from '../modes/template';
 import { getStylusMode } from '../modes/style/stylus';
-import { DocumentContext } from '../types';
+import { DocumentContext, SemanticTokenData } from '../types';
 import { VueInfoService } from '../services/vueInfoService';
 import { DependencyService } from '../services/dependencyService';
 import { nullMode } from '../modes/nullMode';
@@ -74,6 +74,7 @@ export interface LanguageMode {
   getColorPresentations?(document: TextDocument, color: Color, range: Range): ColorPresentation[];
   getFoldingRanges?(document: TextDocument): FoldingRange[];
   getRenameFileEdit?(renames: FileRename): TextDocumentEdit[];
+  getSemanticTokens?(document: TextDocument, range?: Range): SemanticTokenData[] | null;
 
   onDocumentChanged?(filePath: string): void;
   onDocumentRemoved(document: TextDocument): void;

server/src/modes/script/javascript.ts

@@ -44,7 +44,13 @@ import { BasicComponentInfo, VLSFormatConfig } from '../../config';
 import { VueInfoService } from '../../services/vueInfoService';
 import { getComponentInfo } from './componentInfo';
 import { DependencyService, RuntimeLibrary } from '../../services/dependencyService';
-import { CodeActionData, CodeActionDataKind, OrganizeImportsActionData, RefactorActionData } from '../../types';
+import {
+  CodeActionData,
+  CodeActionDataKind,
+  OrganizeImportsActionData,
+  RefactorActionData,
+  SemanticTokenOffsetData
+} from '../../types';
 import { IServiceHost } from '../../services/typescriptService/serviceHost';
 import {
   isVirtualVueTemplateFile,
@@ -57,11 +63,18 @@ import { isVCancellationRequested, VCancellationToken } from '../../utils/cancel
 import { EnvironmentService } from '../../services/EnvironmentService';
 import { getCodeActionKind } from './CodeActionKindConverter';
 import { FileRename } from 'vscode-languageserver';
+import {
+  addCompositionApiRefTokens,
+  getTokenModifierFromClassification,
+  getTokenTypeFromClassification
+} from './semanticToken';
 
 // Todo: After upgrading to LS server 4.0, use CompletionContext for filtering trigger chars
 // https://microsoft.github.io/language-server-protocol/specification#completion-request-leftwards_arrow_with_hook
 const NON_SCRIPT_TRIGGERS = ['<', '*', ':'];
 
+const SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT = 80000;
+
 export async function getJavascriptMode(
   serviceHost: IServiceHost,
   env: EnvironmentService,
@@ -788,6 +801,64 @@ export async function getJavascriptMode(
 
       return textDocumentEdit;
     },
+    getSemanticTokens(doc: TextDocument, range?: Range) {
+      const { scriptDoc, service } = updateCurrentVueTextDocument(doc);
+      const scriptText = scriptDoc.getText();
+      if (scriptText.trim().length > SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT) {
+        return null;
+      }
+
+      const fileFsPath = getFileFsPath(doc.uri);
+      const textSpan = range
+        ? convertTextSpan(range, scriptDoc)
+        : {
+            start: 0,
+            length: scriptText.length
+          };
+      const { spans } = service.getEncodedSemanticClassifications(
+        fileFsPath,
+        textSpan,
+        tsModule.SemanticClassificationFormat.TwentyTwenty
+      );
+
+      const data: SemanticTokenOffsetData[] = [];
+      let index = 0;
+
+      while (index < spans.length) {
+        // [start, length, encodedClassification, start2, length2, encodedClassification2]
+        const start = spans[index++];
+        const length = spans[index++];
+        const encodedClassification = spans[index++];
+        const classificationType = getTokenTypeFromClassification(encodedClassification);
+        if (classificationType < 0) {
+          continue;
+        }
+
+        const modifierSet = getTokenModifierFromClassification(encodedClassification);
+
+        data.push({
+          start,
+          length,
+          classificationType,
+          modifierSet
+        });
+      }
+
+      const program = service.getProgram();
+      if (program) {
+        addCompositionApiRefTokens(tsModule, program, fileFsPath, data);
+      }
+
+      return data.map(({ start, ...rest }) => {
+        const startPosition = scriptDoc.positionAt(start);
+
+        return {
+          ...rest,
+          line: startPosition.line,
+          character: startPosition.character
+        };
+      });
+    },
     dispose() {
       jsDocuments.dispose();
     }
@@ -1114,3 +1185,13 @@ function getFoldingRangeKind(span: ts.OutliningSpan): FoldingRangeKind | undefin
       return undefined;
   }
 }
+
+function convertTextSpan(range: Range, doc: TextDocument): ts.TextSpan {
+  const start = doc.offsetAt(range.start);
+  const end = doc.offsetAt(range.end);
+
+  return {
+    start,
+    length: end - start
+  };
+}

server/src/modes/script/semanticToken.ts

@@ -0,0 +1,149 @@
+import ts from 'typescript';
+import { SemanticTokensLegend, SemanticTokenModifiers, SemanticTokenTypes } from 'vscode-languageserver';
+import { RuntimeLibrary } from '../../services/dependencyService';
+import { SemanticTokenOffsetData } from '../../types';
+
+/* tslint:disable:max-line-length */
+/**
+ * extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9
+ * so that we don't have to map it into our own legend
+ */
+export const enum TokenType {
+  class,
+  enum,
+  interface,
+  namespace,
+  typeParameter,
+  type,
+  parameter,
+  variable,
+  enumMember,
+  property,
+  function,
+  member
+}
+
+/* tslint:disable:max-line-length */
+/**
+ * adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13
+ * so that we don't have to map it into our own legend
+ */
+export const enum TokenModifier {
+  declaration,
+  static,
+  async,
+  readonly,
+  defaultLibrary,
+  local,
+
+  // vue composition api
+  refValue
+}
+
+export function getSemanticTokenLegends(): SemanticTokensLegend {
+  const tokenModifiers: string[] = [];
+
+  ([
+    [TokenModifier.declaration, SemanticTokenModifiers.declaration],
+    [TokenModifier.static, SemanticTokenModifiers.static],
+    [TokenModifier.async, SemanticTokenModifiers.async],
+    [TokenModifier.readonly, SemanticTokenModifiers.readonly],
+    [TokenModifier.defaultLibrary, SemanticTokenModifiers.defaultLibrary],
+    [TokenModifier.local, 'local'],
+
+    // vue
+    [TokenModifier.refValue, 'refValue']
+  ] as const).forEach(([tsModifier, legend]) => (tokenModifiers[tsModifier] = legend));
+
+  const tokenTypes: string[] = [];
+
+  ([
+    [TokenType.class, SemanticTokenTypes.class],
+    [TokenType.enum, SemanticTokenTypes.enum],
+    [TokenType.interface, SemanticTokenTypes.interface],
+    [TokenType.namespace, SemanticTokenTypes.namespace],
+    [TokenType.typeParameter, SemanticTokenTypes.typeParameter],
+    [TokenType.type, SemanticTokenTypes.type],
+    [TokenType.parameter, SemanticTokenTypes.parameter],
+    [TokenType.variable, SemanticTokenTypes.variable],
+    [TokenType.enumMember, SemanticTokenTypes.enumMember],
+    [TokenType.property, SemanticTokenTypes.property],
+    [TokenType.function, SemanticTokenTypes.function],
+
+    // member is renamed to method in vscode codebase to match LSP default
+    [TokenType.member, SemanticTokenTypes.method]
+  ] as const).forEach(([tokenType, legend]) => (tokenTypes[tokenType] = legend));
+
+  return {
+    tokenModifiers,
+    tokenTypes
+  };
+}
+
+export function getTokenTypeFromClassification(tsClassification: number): number {
+  return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
+}
+
+export function getTokenModifierFromClassification(tsClassification: number) {
+  return tsClassification & TokenEncodingConsts.modifierMask;
+}
+
+const enum TokenEncodingConsts {
+  typeOffset = 8,
+  modifierMask = (1 << typeOffset) - 1
+}
+
+export function addCompositionApiRefTokens(
+  tsModule: RuntimeLibrary['typescript'],
+  program: ts.Program,
+  fileFsPath: string,
+  exists: SemanticTokenOffsetData[]
+): void {
+  const sourceFile = program.getSourceFile(fileFsPath);
+
+  if (!sourceFile) {
+    return;
+  }
+
+  const typeChecker = program.getTypeChecker();
+
+  walk(sourceFile, node => {
+    if (!ts.isIdentifier(node) || node.text !== 'value' || !ts.isPropertyAccessExpression(node.parent)) {
+      return;
+    }
+    const propertyAccess = node.parent;
+
+    let parentSymbol = typeChecker.getTypeAtLocation(propertyAccess.expression).symbol;
+
+    if (parentSymbol.flags & tsModule.SymbolFlags.Alias) {
+      parentSymbol = typeChecker.getAliasedSymbol(parentSymbol);
+    }
+
+    if (parentSymbol.name !== 'Ref') {
+      return;
+    }
+
+    const start = node.getStart();
+    const length = node.getWidth();
+    const exist = exists.find(token => token.start === start && token.length === length);
+    const encodedModifier = 1 << TokenModifier.refValue;
+
+    if (exist) {
+      exist.modifierSet |= encodedModifier;
+    } else {
+      exists.push({
+        classificationType: TokenType.property,
+        length: node.getEnd() - node.getStart(),
+        modifierSet: encodedModifier,
+        start: node.getStart()
+      });
+    }
+  });
+}
+
+function walk(node: ts.Node, callback: (node: ts.Node) => void) {
+  node.forEachChild(child => {
+    callback(child);
+    walk(child, callback);
+  });
+}

server/src/services/projectService.ts

@@ -22,6 +22,10 @@ import {
   FoldingRangeParams,
   Hover,
   Location,
+  SemanticTokens,
+  SemanticTokensBuilder,
+  SemanticTokensParams,
+  SemanticTokensRangeParams,
   SignatureHelp,
   SymbolInformation,
   TextDocumentEdit,
@@ -33,7 +37,7 @@ import { URI } from 'vscode-uri';
 import { LanguageId } from '../embeddedSupport/embeddedSupport';
 import { LanguageMode, LanguageModes } from '../embeddedSupport/languageModes';
 import { NULL_COMPLETION, NULL_HOVER, NULL_SIGNATURE } from '../modes/nullMode';
-import { DocumentContext, CodeActionData } from '../types';
+import { DocumentContext, CodeActionData, SemanticTokenData } from '../types';
 import { VCancellationToken } from '../utils/cancellationToken';
 import { getFileFsPath } from '../utils/paths';
 import { DependencyService } from './dependencyService';
@@ -60,6 +64,7 @@ export interface ProjectService {
   onCodeAction(params: CodeActionParams): Promise<CodeAction[]>;
   onCodeActionResolve(action: CodeAction): Promise<CodeAction>;
   onWillRenameFile(fileRename: FileRename): Promise<TextDocumentEdit[]>;
+  onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams): Promise<SemanticTokens | null>;
   doValidate(doc: TextDocument, cancellationToken?: VCancellationToken): Promise<Diagnostic[] | null>;
   dispose(): Promise<void>;
 }
@@ -336,6 +341,38 @@ export async function createProjectService(
 
       return textDocumentEdit ?? [];
     },
+    async onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams) {
+      if (!env.getConfig().vetur.languageFeatures.semanticTokens) {
+        return {
+          data: []
+        };
+      }
+
+      const { textDocument } = params;
+      const range = 'range' in params ? params.range : undefined;
+      const doc = documentService.getDocument(textDocument.uri)!;
+      const modes = languageModes.getAllLanguageModeRangesInDocument(doc);
+      const data: SemanticTokenData[] = [];
+
+      for (const mode of modes) {
+        const tokenData = mode.mode.getSemanticTokens?.(doc, range);
+        // all or nothing
+        if (tokenData === null) {
+          return null;
+        }
+        data.push(...(tokenData ?? []));
+      }
+
+      const builder = new SemanticTokensBuilder();
+      const sorted = data.sort((a, b) => {
+        return a.line - b.line || a.character - b.character;
+      });
+      sorted.forEach(token =>
+        builder.push(token.line, token.character, token.length, token.classificationType, token.modifierSet)
+      );
+
+      return builder.build();
+    },
     async doValidate(doc: TextDocument, cancellationToken?: VCancellationToken) {
       const diagnostics: Diagnostic[] = [];
       if (doc.languageId === 'vue') {