[trie3] do not store references to EOW

streetsidesoftware · Dec 19, 2019 · 0d33de6 · 0d33de6
1 parent 8dd70cb
commit 0d33de6
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 17 deletions.
diff --git a/packages/cspell-trie-lib/Samples/sampleV3.trie b/packages/cspell-trie-lib/Samples/sampleV3.trie
@@ -4,18 +4,18 @@ base=10
 # Sample Words
 # Data:
 __DATA__
-Big Apple$<<<<<<<<races\: \{\}\[\]\(\)#9;<<<<<<<<<<<<<<
-New York#9;<<<<<<<umbers \0\1\2\3\4\5\6\7\8\9#9;<<<<<<<<<<<<<<<<<<
-app#7;<<rrow \<#9;<<<<<<<
+Big Apple$<<<<<<<<races\: \{\}\[\]\(\)$<<<<<<<<<<<<<<
+New York$<<<<<<<umbers \0\1\2\3\4\5\6\7\8\9$<<<<<<<<<<<<<<<<<<
+app#7;<<rrow \<$<<<<<<<
 
 big ap#46;<<<<<<
-eol \\n#9;<<<w \$#9;<<<<scape \\\#9;<<<<<<<<
-fun journey#9;<<<<<<<wal#28;<<<<<<<
-journalism#9;<tic#9;<<$<<<s#9;<$<<eyer#9;<<man#9;<
-<e#96;<<$<<<<ste#94;<ing#9;<<<$<<<vialit#78;<<$<<<<wly#9;<$<<yfuller#9;<st#9;<<<<ness#9;<<<<$<<<lessnes#117;<<<$<<<<ousn#122;<$<<<ridde#96;<<er#9;<$<in#100;
+eol \\n$<<<w \$$<<<<scape \\\$<<<<<<<<
+fun journey$<<<<<<<wal#28;<<<<<<<
+journalism$<tic$<<$<<<s$<$<<eyer$<<man$<
+<e#96;<<$<<<<ste#94;<ing$<<<$<<<vialit#78;<<$<<<<wly$<$<<yfuller$<st$<<<<ness$<<<<$<<<lessnes#117;<<<$<<<<ousn#122;<$<<<ridde#96;<<er$<$<in#100;
 <<<<od#8;<<<stic#28;<<<<$<<<
-lifted#9;<r#9;<<i#132;<s#9;<$<<<ong w#79;<<<<<<
-ref \##9;<<<<<
-talke#141;<i#132;<s#9;<$<<<<
+lifted$<r$<<i#132;<s$<$<<<ong w#79;<<<<<<
+ref \#$<<<<<
+talke#141;<i#132;<s$<$<<<<
 walk#153;<<<<
 
diff --git a/packages/cspell-trie-lib/src/lib/importExportV3.ts b/packages/cspell-trie-lib/src/lib/importExportV3.ts
@@ -68,9 +68,8 @@ export function serializeTrie(root: TrieNode, options: ExportOptions | number =
     function *walk(node: TrieNode, depth: number): Generator<string> {
         const r = cache.get(node);
         if (r !== undefined) {
-            // EOW is only possible if children are yielded first.
-            // yield (node.f && !node.c) ? EOW : ref(r);
-            yield ref(r);
+            yield (node.f && !node.c) ? EOW : ref(r);
+            // yield ref(r);
             return;
         }
         cache.set(node, count++);
@@ -83,7 +82,7 @@ export function serializeTrie(root: TrieNode, options: ExportOptions | number =
                 if (depth === 0) yield EOL;
             }
         }
-        // Output EOW after children so it can be optimized.
+        // Output EOW after children so it can be optimized on read
         if (node.f) {
             yield EOW;
             // yield EOL;
@@ -112,7 +111,6 @@ interface ReduceResults {
 
 type Reducer = (acc: ReduceResults, s: string) => ReduceResults;
 
-const eow: TrieNode = Object.freeze({ f: 1 });
 
 export function importTrie(linesX: IterableLike<string>): TrieNode {
     let radix = 16;
@@ -155,6 +153,8 @@ export function importTrie(linesX: IterableLike<string>): TrieNode {
 }
 
 function parseStream(radix: number): Reducer {
+    let eow: TrieNode | undefined;
+
     function parseReference(acc: ReduceResults, _: string): ReduceResults {
         let ref = '';
 
@@ -164,7 +164,7 @@ function parseStream(radix: number): Reducer {
                 const r = parseInt(ref, radix);
                 const top = stack[stack.length - 1];
                 const p = stack[stack.length - 2].node;
-                p.c?.set(top.s, nodes[r]);
+                p.c!.set(top.s, nodes[r]);
                 return { root, nodes, stack, parser: undefined };
             }
             ref = ref + s;
@@ -211,7 +211,14 @@ function parseStream(radix: number): Reducer {
         const node = top.node;
         node.f = FLAG_WORD;
         if (!node.c) {
-            top.node = eow;
+            if (eow) {
+                top.node = eow;
+                nodes.pop();
+                const p = stack[stack.length - 2].node;
+                p.c!.set(top.s, eow);
+            } else {
+                eow = node;
+            }
         }
         return { root, nodes, stack, parser };
     }