refactor(tokenizer): Use EntityDecoder (#1480)

package.json

@@ -64,7 +64,7 @@
         "domelementtype": "^2.3.0",
         "domhandler": "^5.0.3",
         "domutils": "^3.0.1",
-        "entities": "^4.4.0"
+        "entities": "^4.5.0"
     },
     "devDependencies": {
         "@types/jest": "^29.5.0",

src/Parser.ts

@@ -251,15 +251,10 @@ export class Parser implements Callbacks {
     }
 
     /** @internal */
-    ontextentity(cp: number): void {
-        /*
-         * Entities can be emitted on the character, or directly after.
-         * We use the section start here to get accurate indices.
-         */
-        const index = this.tokenizer.getSectionStart();
-        this.endIndex = index - 1;
+    ontextentity(cp: number, endIndex: number): void {
+        this.endIndex = endIndex - 1;
         this.cbs.ontext?.(fromCodePoint(cp));
-        this.startIndex = index;
+        this.startIndex = endIndex;
     }
 
     protected isVoidElement(name: string): boolean {

src/Tokenizer.spec.ts

@@ -1,10 +1,10 @@
 import { Tokenizer } from "./index.js";
 import type { Callbacks } from "./Tokenizer.js";
 
-function tokenize(data: string) {
+function tokenize(data: string, options = {}) {
     const log: unknown[][] = [];
     const tokenizer = new Tokenizer(
-        {},
+        options,
         new Proxy(
             {},
             {
@@ -56,6 +56,28 @@ describe("Tokenizer", () => {
         });
     });
 
+    describe("should handle entities", () => {
+        it("for XML entities", () =>
+            expect(
+                tokenize("&>&amp<üa&#x62c&#100&#101", {
+                    xmlMode: true,
+                })
+            ).toMatchSnapshot());
+
+        it("for entities in attributes (#276)", () =>
+            expect(
+                tokenize(
+                    '<img src="?&image_uri=1&&image;=2&image=3"/>?&image_uri=1&&image;=2&image=3'
+                )
+            ).toMatchSnapshot());
+
+        it("for trailing legacy entity", () =>
+            expect(tokenize("&timesbar;&timesbar")).toMatchSnapshot());
+
+        it("for multi-byte entities", () =>
+            expect(tokenize("&NotGreaterFullEqual;")).toMatchSnapshot());
+    });
+
     it("should not lose data when pausing", () => {
         const log: unknown[][] = [];
         const tokenizer = new Tokenizer(
@@ -75,7 +97,8 @@ describe("Tokenizer", () => {
             ) as Callbacks
         );
 
-        tokenizer.write("&amp; it up!");
+        tokenizer.write("&am");
+        tokenizer.write("p; it up!");
         tokenizer.resume();
         tokenizer.resume();