Allow opting in to using the Roslyn tokenizer in Razor files

CHANGELOG.md

@@ -4,6 +4,8 @@
 - Debug from .csproj and .sln [#5876](https://github.com/dotnet/vscode-csharp/issues/5876)
 
 # 2.57.x
+* Update Razor to 9.0.0-preview.24561.3 (PR: [#7748](https://github.com/dotnet/vscode-csharp/pull/7748))
+  * Add feature flag to turn on the new Roslyn tokenizer (PR: [#11185](https://github.com/dotnet/razor/pull/11185))
 
 # 2.56.x
 * Update Roslyn to 4.13.0-2.24561.3 (PR: [#7765](https://github.com/dotnet/vscode-csharp/pull/7765))
@@ -28,7 +30,7 @@
   * Reduce memory and CPU costs due to SegmentedList usage (PR: [#75661](https://github.com/dotnet/roslyn/pull/75661))
 * Bump xamltools to 17.13.35506.24 (PR: [#7740](https://github.com/dotnet/vscode-csharp/pull/7740))
 * Bump xamltools to 17.13.35507.225 (PR: [#7755](https://github.com/dotnet/vscode-csharp/pull/7755))
-  * XAML IntelliseSense completions for Image.Source 
+  * XAML IntelliseSense completions for Image.Source
 
 # 2.55.x
 * Update Razor to 9.0.0-preview.24557.10 (PR: [#7757](https://github.com/dotnet/vscode-csharp/pull/7757))

package.json

@@ -39,7 +39,7 @@
   "defaults": {
     "roslyn": "4.13.0-2.24561.3",
     "omniSharp": "1.39.11",
-    "razor": "9.0.0-preview.24557.11",
+    "razor": "9.0.0-preview.24561.3",
     "razorOmnisharp": "7.0.0-preview.23363.1",
     "xamlTools": "17.13.35507.225"
   },
@@ -1528,6 +1528,13 @@
             "description": "%configuration.razor.languageServer.forceRuntimeCodeGeneration%",
             "order": 90
           },
+          "razor.languageServer.useRoslynTokenizer": {
+            "type": "boolean",
+            "scope": "machine-overridable",
+            "default": false,
+            "markdownDescription": "%configuration.razor.languageServer.useRoslynTokenizer%",
+            "order": 90
+          },
           "razor.languageServer.suppressLspErrorToasts": {
             "type": "boolean",
             "default": true,

package.nls.json

@@ -127,6 +127,12 @@
   "configuration.razor.languageServer.debug": "Specifies whether to wait for debug attach when launching the language server.",
   "configuration.razor.server.trace": "Specifies the logging level to use for the Razor server.",
   "configuration.razor.languageServer.forceRuntimeCodeGeneration": "(EXPERIMENTAL) Enable combined design time/runtime code generation for Razor files",
+  "configuration.razor.languageServer.useRoslynTokenizer": {
+    "message": "(EXPERIMENTAL) Use the C# tokenizer for Razor files in the IDE. Enables some new C# features, like interpolated and raw strings, in Razor files opened in Visual Studio Code. This matches using `<features>use-roslyn-tokenizer</feature>` in a `.csproj` file for command line builds, and may result in inconsistencies if this option and your project files do not match.",
+    "comment": [
+      "Markdown text between `` should not be translated or localized (they represent literal text) and the capitalization, spacing, and punctuation (including the ``) should not be altered."
+    ]
+  },
   "configuration.razor.languageServer.suppressLspErrorToasts": "Suppresses error toasts from showing up if the server encounters a recoverable error.",
   "debuggers.coreclr.configurationSnippets.label.console-local": ".NET: Launch Executable file (Console)",
   "debuggers.coreclr.configurationSnippets.label.web-local": ".NET: Launch Executable file (Web)",

src/razor/src/razorLanguageServerClient.ts

@@ -271,6 +271,11 @@ export class RazorLanguageServerClient implements vscode.Disposable {
                 args.push('true');
             }
 
+            if (options.useRoslynTokenizer) {
+                args.push('--UseRoslynTokenizer');
+                args.push('true');
+            }
+
             if (this.telemetryExtensionDllPath.length > 0) {
                 args.push('--telemetryLevel', this.vscodeTelemetryReporter.telemetryLevel);
                 args.push('--sessionId', getSessionId());

src/razor/src/razorLanguageServerOptions.ts

@@ -13,5 +13,6 @@ export interface RazorLanguageServerOptions {
     logLevel: LogLevel;
     usingOmniSharp: boolean;
     forceRuntimeCodeGeneration: boolean;
+    useRoslynTokenizer: boolean;
     suppressErrorToasts: boolean;
 }

src/razor/src/razorLanguageServerOptionsResolver.ts

@@ -25,6 +25,7 @@ export function resolveRazorLanguageServerOptions(
     const usingOmniSharp =
         !getCSharpDevKit() && vscodeApi.workspace.getConfiguration().get<boolean>('dotnet.server.useOmnisharp');
     const forceRuntimeCodeGeneration = serverConfig.get<boolean>('forceRuntimeCodeGeneration');
+    const useRoslynTokenizer = serverConfig.get<boolean>('useRoslynTokenizer');
     const suppressErrorToasts = serverConfig.get<boolean>('suppressLspErrorToasts');
 
     return {
@@ -34,6 +35,7 @@ export function resolveRazorLanguageServerOptions(
         outputChannel: logger.outputChannel,
         usingOmniSharp,
         forceRuntimeCodeGeneration,
+        useRoslynTokenizer,
         suppressErrorToasts,
     } as RazorLanguageServerOptions;
 }