feat(2d): simplify highlighters (#1002)

packages/2d/src/lib/code/CodeHighlighter.ts

@@ -53,9 +53,8 @@ export interface CodeHighlighter<T = unknown> {
    * highlighted.
    *
    * @param code - The code to prepare.
-   * @param dialect - The language in which the code is written.
    */
-  prepare(code: string, dialect: string): T;
+  prepare(code: string): T;
 
   /**
    * Highlights the code at the given index.
@@ -69,7 +68,6 @@ export interface CodeHighlighter<T = unknown> {
    * Tokenize the code.
    *
    * @param code - The code to tokenize.
-   * @param dialect - The language in which the code is written.
    */
-  tokenize(code: string, dialect: string): string[];
+  tokenize(code: string): string[];
 }

packages/2d/src/lib/code/CodeSignal.ts

@@ -80,13 +80,11 @@ export class CodeSignalContext<TOwner>
   public constructor(
     initial: SignalValue<PossibleCodeScope>,
     owner: TOwner,
-    private readonly highlighter?: SignalValue<CodeHighlighter>,
-    private readonly dialect?: SignalValue<string>,
+    private readonly highlighter?: SignalValue<CodeHighlighter | null>,
   ) {
     super(initial, deepLerp, owner);
     if (owner instanceof Code) {
       this.highlighter ??= owner.highlighter;
-      this.dialect ??= owner.dialect;
     }
     Object.defineProperty(this.invokable, 'edit', {
       value: this.edit.bind(this),
@@ -115,15 +113,14 @@ export class CodeSignalContext<TOwner>
   ): ThreadGenerator {
     let tokenize = defaultTokenize;
     const highlighter = unwrap(this.highlighter);
-    const dialect = unwrap(this.dialect);
-    if (highlighter && dialect) {
+    if (highlighter) {
       yield (async () => {
         do {
           await DependencyContext.consumePromises();
           highlighter.initialize();
         } while (DependencyContext.hasPromises());
       })();
-      tokenize = (input: string) => highlighter.tokenize(input, dialect);
+      tokenize = (input: string) => highlighter.tokenize(input);
     }
 
     this.progress(0);

packages/2d/src/lib/code/LezerHighlighter.ts

@@ -1,9 +1,8 @@
 import {HighlightStyle} from '@codemirror/language';
 import {Parser, SyntaxNode, Tree} from '@lezer/common';
 import {highlightTree} from '@lezer/highlight';
-import {useLogger} from '@motion-canvas/core';
 import {CodeHighlighter, HighlightResult} from './CodeHighlighter';
-import {defaultTokenize} from './CodeTokenizer';
+import {DefaultHighlightStyle} from './DefaultHighlightStyle';
 
 interface LezerCache {
   tree: Tree;
@@ -12,15 +11,13 @@ interface LezerCache {
 }
 
 export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
-  private static readonly parserMap = new Map<string, Parser>();
   private static classRegex = /\.(\S+).*color:([^;]+)/;
   private readonly classLookup = new Map<string, string>();
 
-  public static registerParser(parser: Parser, dialect = ''): void {
-    this.parserMap.set(dialect, parser);
-  }
-
-  public constructor(private readonly style: HighlightStyle) {
+  public constructor(
+    private readonly parser: Parser,
+    private readonly style: HighlightStyle = DefaultHighlightStyle,
+  ) {
     for (const rule of this.style.module?.getRules().split('\n') ?? []) {
       const match = rule.match(LezerHighlighter.classRegex);
       if (!match) {
@@ -37,17 +34,9 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
     return true;
   }
 
-  public prepare(code: string, dialect: string): LezerCache | null {
-    const parser = LezerHighlighter.parserMap.get(dialect);
-    if (!parser) {
-      if (dialect !== '') {
-        useLogger().warn(`No parser found for dialect: ${dialect}`);
-      }
-      return null;
-    }
-
+  public prepare(code: string): LezerCache | null {
     const colorLookup = new Map<string, string>();
-    const tree = parser.parse(code);
+    const tree = this.parser.parse(code);
     highlightTree(tree, this.style, (from, to, classes) => {
       const color = this.classLookup.get(classes);
       if (!color) {
@@ -97,16 +86,8 @@ export class LezerHighlighter implements CodeHighlighter<LezerCache | null> {
     };
   }
 
-  public tokenize(code: string, dialect: string): string[] {
-    const parser = LezerHighlighter.parserMap.get(dialect);
-    if (!parser) {
-      if (dialect !== '') {
-        useLogger().warn(`No parser found for dialect: ${dialect}`);
-      }
-      return defaultTokenize(code);
-    }
-
-    const tree = parser.parse(code);
+  public tokenize(code: string): string[] {
+    const tree = this.parser.parse(code);
     const cursor = tree.cursor();
     const tokens: string[] = [];
     let current = 0;

packages/2d/src/lib/components/Code.ts

@@ -2,7 +2,6 @@ import {
   BBox,
   createSignal,
   experimentalLog,
-  lazy,
   map,
   SerializedVector2,
   Signal,
@@ -25,10 +24,8 @@ import {
   CodeSignal,
   codeSignal,
   CodeSignalContext,
-  DefaultHighlightStyle,
   findAllCodeRanges,
   isPointInCodeSelection,
-  LezerHighlighter,
   lines,
   parseCodeSelection,
   PossibleCodeScope,
@@ -81,10 +78,6 @@ export interface CodeProps extends ShapeProps {
    * {@inheritDoc Code.highlighter}
    */
   highlighter?: SignalValue<CodeHighlighter | null>;
-  /**
-   * {@inheritDoc Code.dialect}
-   */
-  dialect?: SignalValue<string>;
   /**
    * {@inheritDoc Code.code}
    */
@@ -145,58 +138,19 @@ export class Code extends Shape {
    *
    * @param initial - The initial code.
    * @param highlighter - Custom highlighter to use.
-   * @param dialect - Custom dialect to use.
    */
   public static createSignal(
     initial: SignalValue<PossibleCodeScope>,
     highlighter?: SignalValue<CodeHighlighter>,
-    dialect?: SignalValue<string>,
   ): CodeSignal<void> {
     return new CodeSignalContext<void>(
       initial,
       undefined,
       highlighter,
-      dialect,
     ).toSignal();
   }
 
-  @lazy(() => new LezerHighlighter(DefaultHighlightStyle))
-  public static readonly defaultHighlighter: LezerHighlighter;
-
-  /**
-   * The dialect to use for highlighting the code.
-   *
-   * @remarks
-   * This value will be passed to the {@link code.CodeHighlighter}
-   * defined by the {@link highlighter} property. Different highlighters may use
-   * it differently.
-   *
-   * The default {@link code.LezerHighlighter} uses it to select
-   * the language parser to use. The parser for the given dialect can be
-   * registered as follows:
-   * ```tsx
-   * // Import the lezer parser:
-   * import {parser} from '@lezer/javascript';
-   *
-   * // Register it in the highlighter:
-   * LezerHighlighter.registerParser(parser, 'js');
-   *
-   * // Use the dialect in a code node:
-   * <Code dialect="js" code="const a = 7;" />
-   * ```
-   * When no dialect is provided, the highlighter will use the default
-   * parser:
-   * ```tsx
-   * // Register the default parser by omitting the dialect:
-   * LezerHighlighter.registerParser(parser);
-   *
-   * // Code nodes with no dialect will now use the default parser:
-   * <Code code="const a = 7;" />
-   * ```
-   */
-  @initial('')
-  @signal()
-  public declare readonly dialect: SimpleSignal<string, this>;
+  public static defaultHighlighter: CodeHighlighter | null = null;
 
   /**
    * The code highlighter to use for this code node.
@@ -206,7 +160,10 @@ export class Code extends Shape {
    */
   @initial(() => Code.defaultHighlighter)
   @signal()
-  public declare readonly highlighter: SimpleSignal<CodeHighlighter, this>;
+  public declare readonly highlighter: SimpleSignal<
+    CodeHighlighter | null,
+    this
+  >;
 
   /**
    * The code to display.
@@ -327,11 +284,10 @@ export class Code extends Shape {
     const code = this.code();
     const before = resolveScope(code, false);
     const after = resolveScope(code, true);
-    const dialect = this.dialect();
 
     return {
-      before: highlighter.prepare(before, dialect),
-      after: highlighter.prepare(after, dialect),
+      before: highlighter.prepare(before),
+      after: highlighter.prepare(after),
     };
   }
 
@@ -360,7 +316,6 @@ export class Code extends Shape {
       initial,
       this,
       this.highlighter,
-      this.dialect,
     ).toSignal();
   }