diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts index cbcfc36a3d1..da05ad24ec3 100644 --- a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts +++ b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts @@ -157,7 +157,7 @@ describe('Validate Larger Dictionary', () => { const trie = normalizeWordsToTrie(words); expect(isCircular(trie)).toBe(false); const nWords = toArray(legacyNormalizeWords(words)).sort().filter(uniqueFilter(1000)); - const results = iteratorTrieWords(trie).toArray().sort().filter(uniqueFilter(1000)); + const results = [...iteratorTrieWords(trie)].sort().filter(uniqueFilter(1000)); expect(results).toEqual(nWords); }, 60000); @@ -167,11 +167,11 @@ describe('Validate Larger Dictionary', () => { const trie = Trie.consolidate(normalizeWordsToTrie(words)); expect(isCircular(trie)).toBe(false); const nWords = toArray(legacyNormalizeWords(words)).sort().filter(uniqueFilter(1000)); - const results = iteratorTrieWords(trie).toArray().sort(); + const results = [...iteratorTrieWords(trie)].sort(); expect(results).toEqual(nWords); const data = serializeTrie(trie, { base: 40 }); const trie2 = importTrie(data); - const results2 = iteratorTrieWords(trie2).toArray(); + const results2 = [...iteratorTrieWords(trie2)]; expect(results2).toEqual(results); }, 60000); }); diff --git a/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.test.ts b/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.test.ts index f38ed13b7a4..57ba0926fd4 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { createTriFromList } from '../trie-util.js'; +import { createTriFromList } from '../TrieNode/trie-util.js'; import { FastTrieBlob } from './FastTrieBlob.js'; describe('FastTrieBlob', () => { diff --git a/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.ts b/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.ts index 45e396fe168..ddb206e13b3 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBlob/FastTrieBlob.ts @@ -1,4 +1,4 @@ -import type { TrieNode, TrieRoot } from '../TrieNode.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; import { TrieBlob } from './TrieBlob.js'; type FastTrieBlobNode = number[]; diff --git a/packages/cspell-trie-lib/src/lib/TrieBlob/test/en.spec.ts b/packages/cspell-trie-lib/src/lib/TrieBlob/test/en.spec.ts index 4eb6e59b8af..04c4e1ac423 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBlob/test/en.spec.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBlob/test/en.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { readTrie } from '../../../test/dictionaries.test.helper.js'; -import type { TrieNode } from '../../TrieNode.js'; +import type { TrieNode } from '../../TrieNode/TrieNode.js'; function getTrie() { return readTrie('@cspell/dict-en_us/cspell-ext.json'); diff --git a/packages/cspell-trie-lib/src/lib/TrieBlob/test/perfFastTrieBlob.ts b/packages/cspell-trie-lib/src/lib/TrieBlob/test/perfFastTrieBlob.ts index c44c67610d8..f0216da922c 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBlob/test/perfFastTrieBlob.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBlob/test/perfFastTrieBlob.ts @@ -24,10 +24,10 @@ function hasWords(words: string[], method: (word: string) => boolean): boolean { export async function measureFastBlob(which: string | undefined, method: string | undefined) { const trie = await getTrie(); - const words = trie.words().toArray(); + const words = [...trie.words()]; if (filterTest(which, 'blob')) { - const ft = measure('blob.FastTrieBlob \t\t', () => FastTrieBlob.fromTrieRoot(trie.root)); + const ft = measure('blob.FastTrieBlob.fromTrieRoot \t', () => FastTrieBlob.fromTrieRoot(trie.root)); const trieBlob = measure('blob.FastTrieBlob.toTrieBlob \t', () => ft.toTrieBlob()); switch (method) { @@ -52,7 +52,7 @@ export async function measureFastBlob(which: string | undefined, method: string } if (filterTest(which, 'fast')) { - const ft = measure('fast.FastTrieBlob \t\t', () => FastTrieBlob.fromWordList(words)); + const ft = measure('fast.FastTrieBlob.fromWordList \t', () => FastTrieBlob.fromWordList(words)); switch (method) { case 'has': diff --git a/packages/cspell-trie-lib/src/lib/TrieBuilder.test.ts b/packages/cspell-trie-lib/src/lib/TrieBuilder.test.ts index aca632e94fc..3cafc9dbfe8 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBuilder.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBuilder.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; -import { countNodes, isCircular } from './trie-util.js'; import { buildTrie, TrieBuilder } from './TrieBuilder.js'; +import { countNodes, isCircular } from './TrieNode/trie-util.js'; describe('Validate TrieBuilder', () => { test('builder explicit consolidateSuffixes', () => { diff --git a/packages/cspell-trie-lib/src/lib/TrieBuilder.ts b/packages/cspell-trie-lib/src/lib/TrieBuilder.ts index fdd8fe09291..30942d8c1ef 100644 --- a/packages/cspell-trie-lib/src/lib/TrieBuilder.ts +++ b/packages/cspell-trie-lib/src/lib/TrieBuilder.ts @@ -1,8 +1,8 @@ import { consolidate } from './consolidate.js'; import type { PartialTrieOptions, TrieOptions } from './trie.js'; import { Trie } from './trie.js'; -import { createTrieRoot, createTriFromList, trieNodeToRoot } from './trie-util.js'; -import type { TrieNode, TrieRoot } from './TrieNode.js'; +import { createTrieRoot, createTriFromList, trieNodeToRoot } from './TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from './TrieNode/TrieNode.js'; import { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.js'; import { SecondChanceCache } from './utils/secondChanceCache.js'; diff --git a/packages/cspell-trie-lib/src/lib/TrieNode.ts b/packages/cspell-trie-lib/src/lib/TrieNode/TrieNode.ts similarity index 89% rename from packages/cspell-trie-lib/src/lib/TrieNode.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/TrieNode.ts index 2e5a8191cb7..9a722fd2fe7 100644 --- a/packages/cspell-trie-lib/src/lib/TrieNode.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/TrieNode.ts @@ -1,4 +1,4 @@ -import type { PartialWithUndefined } from './types.js'; +import type { PartialWithUndefined } from '../types.js'; export const FLAG_WORD = 1; diff --git a/packages/cspell-trie-lib/src/lib/compoundWalker.test.ts b/packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.test.ts similarity index 97% rename from packages/cspell-trie-lib/src/lib/compoundWalker.test.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.test.ts index 7cf9287ed75..377dbbc8e21 100644 --- a/packages/cspell-trie-lib/src/lib/compoundWalker.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.test.ts @@ -1,10 +1,10 @@ import { describe, expect, test } from 'vitest'; +import { parseDictionary } from '../SimpleDictionaryParser.js'; +import type { Trie } from '../trie.js'; import type { WalkItem, WalkNext } from './compoundWalker.js'; import { compoundWalker, compoundWords } from './compoundWalker.js'; import { findWord } from './find.js'; -import { parseDictionary } from './SimpleDictionaryParser.js'; -import type { Trie } from './trie.js'; // cspell:ignore errorerror diff --git a/packages/cspell-trie-lib/src/lib/compoundWalker.ts b/packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.ts similarity index 98% rename from packages/cspell-trie-lib/src/lib/compoundWalker.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.ts index 23765727968..05b09aa33a1 100644 --- a/packages/cspell-trie-lib/src/lib/compoundWalker.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/compoundWalker.ts @@ -1,4 +1,4 @@ -import type { Trie } from './trie.js'; +import type { Trie } from '../trie.js'; import type { TrieNode } from './TrieNode.js'; export interface WalkItem { diff --git a/packages/cspell-trie-lib/src/lib/find.dutch.test.ts b/packages/cspell-trie-lib/src/lib/TrieNode/find.dutch.test.ts similarity index 98% rename from packages/cspell-trie-lib/src/lib/find.dutch.test.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/find.dutch.test.ts index a35c91504f2..9ae98e10d42 100644 --- a/packages/cspell-trie-lib/src/lib/find.dutch.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/find.dutch.test.ts @@ -2,12 +2,12 @@ import * as fs from 'fs/promises'; import { describe, expect, test } from 'vitest'; import * as zlib from 'zlib'; -import { resolveGlobalDict } from '../test/samples.js'; +import { resolveGlobalDict } from '../../test/samples.js'; +import { importTrie } from '../io/importExport.js'; +import { normalizeWordToLowercase } from '../utils/normalizeWord.js'; import type { FindFullResult, PartialFindOptions } from './find.js'; import { findWord } from './find.js'; -import { importTrie } from './io/importExport.js'; import type { TrieNode } from './TrieNode.js'; -import { normalizeWordToLowercase } from './utils/normalizeWord.js'; const dutchDictionary = resolveGlobalDict('nl_nl.trie.gz'); diff --git a/packages/cspell-trie-lib/src/lib/find.test.ts b/packages/cspell-trie-lib/src/lib/TrieNode/find.test.ts similarity index 99% rename from packages/cspell-trie-lib/src/lib/find.test.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/find.test.ts index f263690392f..381d6c67c4a 100644 --- a/packages/cspell-trie-lib/src/lib/find.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/find.test.ts @@ -1,9 +1,9 @@ import { describe, expect, test } from 'vitest'; +import { parseDictionary } from '../SimpleDictionaryParser.js'; +import { Trie } from '../trie.js'; import type { FindFullResult, PartialFindOptions } from './find.js'; import { __testing__, createFindOptions, findLegacyCompound, findWord } from './find.js'; -import { parseDictionary } from './SimpleDictionaryParser.js'; -import { Trie } from './trie.js'; const findLegacyCompoundWord = __testing__.findLegacyCompoundWord; diff --git a/packages/cspell-trie-lib/src/lib/find.ts b/packages/cspell-trie-lib/src/lib/TrieNode/find.ts similarity index 98% rename from packages/cspell-trie-lib/src/lib/find.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/find.ts index 9ce146bedcf..ae87bbdb9d2 100644 --- a/packages/cspell-trie-lib/src/lib/find.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/find.ts @@ -1,9 +1,9 @@ -import { CASE_INSENSITIVE_PREFIX, COMPOUND_FIX, FORBID_PREFIX } from './constants.js'; +import { CASE_INSENSITIVE_PREFIX, COMPOUND_FIX, FORBID_PREFIX } from '../constants.js'; +import type { PartialWithUndefined } from '../types.js'; +import { memorizeLastCall } from '../utils/memorizeLastCall.js'; +import { mergeDefaults } from '../utils/mergeDefaults.js'; import type { TrieNode, TrieRoot } from './TrieNode.js'; import { FLAG_WORD } from './TrieNode.js'; -import type { PartialWithUndefined } from './types.js'; -import { memorizeLastCall } from './utils/memorizeLastCall.js'; -import { mergeDefaults } from './utils/mergeDefaults.js'; type Root = PartialWithUndefined; diff --git a/packages/cspell-trie-lib/src/lib/trie-util.test.ts b/packages/cspell-trie-lib/src/lib/TrieNode/trie-util.test.ts similarity index 97% rename from packages/cspell-trie-lib/src/lib/trie-util.test.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/trie-util.test.ts index 0fca3f4eb49..d0b48241578 100644 --- a/packages/cspell-trie-lib/src/lib/trie-util.test.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/trie-util.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'vitest'; +import { mergeDefaults } from '../utils/mergeDefaults.js'; import { countNodes, countWords, @@ -9,7 +10,6 @@ import { isCircular, iteratorTrieWords, } from './trie-util.js'; -import { mergeDefaults } from './utils/mergeDefaults.js'; describe('Validate Util Functions', () => { test('createTriFromList', () => { diff --git a/packages/cspell-trie-lib/src/lib/trie-util.ts b/packages/cspell-trie-lib/src/lib/TrieNode/trie-util.ts similarity index 88% rename from packages/cspell-trie-lib/src/lib/trie-util.ts rename to packages/cspell-trie-lib/src/lib/TrieNode/trie-util.ts index 4d52a181a2a..c051aed65df 100644 --- a/packages/cspell-trie-lib/src/lib/trie-util.ts +++ b/packages/cspell-trie-lib/src/lib/TrieNode/trie-util.ts @@ -1,11 +1,10 @@ -import type { Sequence } from 'gensequence'; -import { genSequence } from 'gensequence'; +import { opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync'; +import { mergeOptionalWithDefaults } from '../utils/mergeOptionalWithDefaults.js'; +import { walker } from '../walker/walker.js'; +import type { YieldResult } from '../walker/walkerTypes.js'; import type { PartialTrieOptions, TrieNode, TrieRoot } from './TrieNode.js'; import { FLAG_WORD } from './TrieNode.js'; -import { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.js'; -import { walker } from './walker/walker.js'; -import type { YieldResult } from './walker/walkerTypes.js'; export function insert(text: string, root: TrieNode = {}): TrieNode { let node = root; @@ -40,8 +39,8 @@ export function orderTrie(node: TrieNode): void { /** * Generator an iterator that will walk the Trie parent then children in a depth first fashion that preserves sorted order. */ -export function walk(node: TrieNode): Sequence { - return genSequence(walker(node)); +export function walk(node: TrieNode): Iterable { + return walker(node); } export const iterateTrie = walk; @@ -49,10 +48,12 @@ export const iterateTrie = walk; /** * Generate a Iterator that can walk a Trie and yield the words. */ -export function iteratorTrieWords(node: TrieNode): Sequence { - return walk(node) - .filter((r) => isWordTerminationNode(r.node)) - .map((r) => r.text); +export function iteratorTrieWords(node: TrieNode): Iterable { + return pipe( + walk(node), + opFilter((r) => isWordTerminationNode(r.node)), + opMap((r) => r.text) + ); } export function createTrieRoot(options: PartialTrieOptions): TrieRoot { diff --git a/packages/cspell-trie-lib/src/lib/consolidate.test.ts b/packages/cspell-trie-lib/src/lib/consolidate.test.ts index 378c06a573c..aee55a56792 100644 --- a/packages/cspell-trie-lib/src/lib/consolidate.test.ts +++ b/packages/cspell-trie-lib/src/lib/consolidate.test.ts @@ -5,9 +5,9 @@ import { describe, expect, test } from 'vitest'; import { resolveGlobalSample } from '../test/samples.js'; import { consolidate } from './consolidate.js'; -import { countNodes, createTrieRoot, createTriFromList, iteratorTrieWords } from './trie-util.js'; import { buildTrie } from './TrieBuilder.js'; -import type { TrieNode } from './TrieNode.js'; +import { countNodes, createTrieRoot, createTriFromList, iteratorTrieWords } from './TrieNode/trie-util.js'; +import type { TrieNode } from './TrieNode/TrieNode.js'; const samples = resolveGlobalSample('dicts'); const sampleEnglish = path.join(samples, 'en_US.txt'); diff --git a/packages/cspell-trie-lib/src/lib/consolidate.ts b/packages/cspell-trie-lib/src/lib/consolidate.ts index fe443494e54..6a84d81043b 100644 --- a/packages/cspell-trie-lib/src/lib/consolidate.ts +++ b/packages/cspell-trie-lib/src/lib/consolidate.ts @@ -1,6 +1,6 @@ -import { trieNodeToRoot } from './trie-util.js'; -import type { TrieNode, TrieRoot } from './TrieNode.js'; -import { FLAG_WORD } from './TrieNode.js'; +import { trieNodeToRoot } from './TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from './TrieNode/TrieNode.js'; +import { FLAG_WORD } from './TrieNode/TrieNode.js'; /** * Consolidate to DAWG diff --git a/packages/cspell-trie-lib/src/lib/constants.ts b/packages/cspell-trie-lib/src/lib/constants.ts index ab4457b0fc3..fa34ef08251 100644 --- a/packages/cspell-trie-lib/src/lib/constants.ts +++ b/packages/cspell-trie-lib/src/lib/constants.ts @@ -1,4 +1,4 @@ -import type { TrieOptions } from './TrieNode.js'; +import type { TrieOptions } from './TrieNode/TrieNode.js'; export const COMPOUND_FIX = '+'; export const OPTIONAL_COMPOUND_FIX = '*'; diff --git a/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.test.ts b/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.test.ts index 45fdec1a20a..3a790a0e928 100644 --- a/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.test.ts +++ b/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.test.ts @@ -3,7 +3,7 @@ import { describe, expect, test } from 'vitest'; import { consolidate } from './consolidate.js'; import { convertToTrieRefNodes } from './convertToTrieRefNodes.js'; -import { createTriFromList } from './trie-util.js'; +import { createTriFromList } from './TrieNode/trie-util.js'; import type { TrieRefNode } from './trieRef.js'; describe('Validate convertToTrieRefNodes', () => { diff --git a/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.ts b/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.ts index 0b71801b943..f8f65c7c9b4 100644 --- a/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.ts +++ b/packages/cspell-trie-lib/src/lib/convertToTrieRefNodes.ts @@ -1,7 +1,7 @@ import { genSequence } from 'gensequence'; -import type { TrieNode } from './TrieNode.js'; -import { FLAG_WORD } from './TrieNode.js'; +import type { TrieNode } from './TrieNode/TrieNode.js'; +import { FLAG_WORD } from './TrieNode/TrieNode.js'; import type { TrieRefNode } from './trieRef.js'; const MinReferenceCount = 3; diff --git a/packages/cspell-trie-lib/src/lib/flatten.test.ts b/packages/cspell-trie-lib/src/lib/flatten.test.ts index 5392af83157..896b2235a4b 100644 --- a/packages/cspell-trie-lib/src/lib/flatten.test.ts +++ b/packages/cspell-trie-lib/src/lib/flatten.test.ts @@ -2,7 +2,7 @@ import { genSequence } from 'gensequence'; import { describe, expect, test } from 'vitest'; import { flattenToTrieRefNodeArray, flattenToTrieRefNodeIterable } from './flatten.js'; -import { createTriFromList } from './trie-util.js'; +import { createTriFromList } from './TrieNode/trie-util.js'; import type { TrieRefNode } from './trieRef.js'; describe('Validate Flatten', () => { diff --git a/packages/cspell-trie-lib/src/lib/flatten.ts b/packages/cspell-trie-lib/src/lib/flatten.ts index 8b4fe2c40ed..163042e0b44 100644 --- a/packages/cspell-trie-lib/src/lib/flatten.ts +++ b/packages/cspell-trie-lib/src/lib/flatten.ts @@ -1,4 +1,4 @@ -import type { TrieNode } from './TrieNode.js'; +import type { TrieNode } from './TrieNode/TrieNode.js'; import type { TrieRefNode } from './trieRef.js'; export interface Emitter { diff --git a/packages/cspell-trie-lib/src/lib/index.ts b/packages/cspell-trie-lib/src/lib/index.ts index 5b72bb2d8f2..920782deef3 100644 --- a/packages/cspell-trie-lib/src/lib/index.ts +++ b/packages/cspell-trie-lib/src/lib/index.ts @@ -1,7 +1,6 @@ export { consolidate } from './consolidate.js'; export type { WeightMap } from './distance/index.js'; export { createWeightedMap, editDistance, editDistanceWeighted } from './distance/index.js'; -export type { FindFullResult } from './find.js'; export { ExportOptions, importTrie, serializeTrie } from './io/importExport.js'; export { mapDictionaryInformationToWeightMap } from './mappers/mapDictionaryInfoToWeightMap.js'; export type { SuggestionCostMapDef } from './models/suggestionCostsDef.js'; @@ -25,6 +24,8 @@ export { OPTIONAL_COMPOUND_FIX, Trie, } from './trie.js'; +export { buildTrie, buildTrieFast, TrieBuilder } from './TrieBuilder.js'; +export type { FindFullResult } from './TrieNode/find.js'; export { countNodes, countWords, @@ -40,9 +41,8 @@ export { orderTrie, trieNodeToRoot, walk, -} from './trie-util.js'; -export { buildTrie, buildTrieFast, TrieBuilder } from './TrieBuilder.js'; -export { ChildMap, FLAG_WORD, TrieNode, TrieRoot } from './TrieNode.js'; +} from './TrieNode/trie-util.js'; +export { ChildMap, FLAG_WORD, TrieNode, TrieRoot } from './TrieNode/TrieNode.js'; export { isDefined } from './utils/isDefined.js'; export { mergeDefaults } from './utils/mergeDefaults.js'; export { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.js'; diff --git a/packages/cspell-trie-lib/src/lib/io/importExport.ts b/packages/cspell-trie-lib/src/lib/io/importExport.ts index 9dc1c087231..9945b71e1d9 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExport.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExport.ts @@ -1,8 +1,6 @@ import { toDistributableIterable } from '@cspell/cspell-pipe'; -import type { Sequence } from 'gensequence'; -import { genSequence } from 'gensequence'; -import type { TrieRoot } from '../TrieNode.js'; +import type { TrieRoot } from '../TrieNode/TrieNode.js'; import * as iv1 from './importExportV1.js'; import * as iv2 from './importExportV2.js'; import * as iv3 from './importExportV3.js'; @@ -15,7 +13,7 @@ export interface ExportOptions { addLineBreaksToImproveDiffs?: boolean; } -type Serializer = (root: TrieRoot, options?: number | ExportOptions) => Sequence; +type Serializer = (root: TrieRoot, options?: number | ExportOptions) => Iterable; const serializers: readonly Serializer[] = [ iv1.serializeTrie, @@ -35,7 +33,7 @@ const DEFAULT_VERSION = 3; * Even though it is possible to preserve the trie, dealing with very large tries can consume a lot of memory. * Considering this is the last step before exporting, it was decided to let this be destructive. */ -export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Sequence { +export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Iterable { const version = typeof options !== 'number' && options.version ? options.version : DEFAULT_VERSION; const method = serializers[version]; if (!method) { @@ -44,7 +42,8 @@ export function serializeTrie(root: TrieRoot, options: ExportOptions | number = return method(root, options); } -export function importTrie(lines: Iterable | IterableIterator): TrieRoot { +export function importTrie(lines: Iterable | IterableIterator | string[]): TrieRoot { + const aLines = Array.isArray(lines) ? lines : [...lines]; function parseHeaderRows(headerRows: string[]): number { const header = headerRows.join('\n'); const headerReg = /^\s*TrieXv(\d+)/m; @@ -65,10 +64,10 @@ export function importTrie(lines: Iterable | IterableIterator): return headerRows; } - const input = toDistributableIterable(lines); + const input = toDistributableIterable(aLines); const headerLines = readHeader(input); const version = parseHeaderRows(headerLines); - const stream = genSequence(headerLines).concat(input); + const stream = [...headerLines, ...input]; const method = deserializers[version]; if (!method) { throw new Error(`Unsupported version: ${version}`); diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV1.test.ts b/packages/cspell-trie-lib/src/lib/io/importExportV1.test.ts index 9133d7c556a..864b1c290da 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV1.test.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV1.test.ts @@ -4,7 +4,7 @@ import { describe, expect, test } from 'vitest'; import { resolveSample } from '../../test/samples.js'; import { consolidate } from '../consolidate.js'; import * as Trie from '../index.js'; -import { iteratorTrieWords } from '../trie-util.js'; +import { iteratorTrieWords } from '../TrieNode/trie-util.js'; import { importTrie, serializeTrie } from './importExportV1.js'; describe('Import/Export', () => { diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV1.ts b/packages/cspell-trie-lib/src/lib/io/importExportV1.ts index ed6ba83d802..6c1f17ac909 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV1.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV1.ts @@ -2,9 +2,9 @@ import type { Sequence } from 'gensequence'; import { genSequence } from 'gensequence'; import { convertToTrieRefNodes } from '../convertToTrieRefNodes.js'; -import { trieNodeToRoot } from '../trie-util.js'; -import type { TrieNode, TrieRoot } from '../TrieNode.js'; -import { FLAG_WORD } from '../TrieNode.js'; +import { trieNodeToRoot } from '../TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; +import { FLAG_WORD } from '../TrieNode/TrieNode.js'; import type { TrieRefNode } from '../trieRef.js'; const EOW = '*'; diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV2.ts b/packages/cspell-trie-lib/src/lib/io/importExportV2.ts index a4f6439b13e..9ca7e9403c8 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV2.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV2.ts @@ -1,9 +1,9 @@ import type { Sequence } from 'gensequence'; import { genSequence } from 'gensequence'; -import { trieNodeToRoot } from '../trie-util.js'; -import type { TrieNode, TrieRoot } from '../TrieNode.js'; -import { FLAG_WORD } from '../TrieNode.js'; +import { trieNodeToRoot } from '../TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; +import { FLAG_WORD } from '../TrieNode/TrieNode.js'; const EOW = '*'; export const DATA = '__DATA__'; diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV3.test.ts b/packages/cspell-trie-lib/src/lib/io/importExportV3.test.ts index 979906aebe6..ad470392f14 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV3.test.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV3.test.ts @@ -5,7 +5,7 @@ import { describe, expect, test } from 'vitest'; import { resolveSample as resolveSamplePath } from '../../test/samples.js'; import { consolidate } from '../consolidate.js'; import * as Trie from '../index.js'; -import type { TrieNode } from '../TrieNode.js'; +import type { TrieNode } from '../TrieNode/TrieNode.js'; import { importTrie, serializeTrie } from './importExportV3.js'; const sampleFile = resolveSamplePath('sampleV3.trie'); diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV3.ts b/packages/cspell-trie-lib/src/lib/io/importExportV3.ts index c1d121f02be..2c388edf9ff 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV3.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV3.ts @@ -1,9 +1,8 @@ -import type { Sequence } from 'gensequence'; -import { genSequence } from 'gensequence'; +import { opAppend, opConcatMap, pipe, reduce } from '@cspell/cspell-pipe/sync'; -import { trieNodeToRoot } from '../trie-util.js'; -import type { TrieNode, TrieRoot } from '../TrieNode.js'; -import { FLAG_WORD } from '../TrieNode.js'; +import { trieNodeToRoot } from '../TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; +import { FLAG_WORD } from '../TrieNode/TrieNode.js'; import { bufferLines } from '../utils/bufferLines.js'; const EOW = '$'; // End of word @@ -33,11 +32,11 @@ const WORDS_PER_LINE = 20; export const DATA = '__DATA__'; -function generateHeader(base: number, comment: string): Sequence { +function generateHeader(base: number, comment: string): Iterable { const header = ['#!/usr/bin/env cspell-trie reader', 'TrieXv3', 'base=' + base] .concat(comment ? comment.split('\n').map((a) => '# ' + a) : []) .concat(['# Data:', DATA]); - return genSequence(header).map((a) => a + '\n'); + return header.map((a) => a + '\n'); } export interface ExportOptions { @@ -58,7 +57,7 @@ export interface ExportOptions { /** * Serialize a TrieRoot. */ -export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Sequence { +export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Iterable { options = typeof options === 'number' ? { base: options, addLineBreaksToImproveDiffs: false } : options; const { base = 16, comment = '', addLineBreaksToImproveDiffs: addBreaks = true } = options; const radix = base > 36 ? 36 : base < 10 ? 10 : base; @@ -167,7 +166,7 @@ export function serializeTrie(root: TrieRoot, options: ExportOptions | number = return rr; } - return generateHeader(radix, comment).concat(bufferLines(serialize(root), 1200, '')); + return pipe(generateHeader(radix, comment), opAppend(bufferLines(serialize(root), 1200, ''))); } function* toIterableIterator(iter: Iterable): IterableIterator { @@ -226,14 +225,19 @@ export function importTrie(linesX: Iterable | string): TrieRoot { readHeader(iter); - const n = genSequence(iter) - .concatMap((a) => a.split('')) - .reduce(parseStream(radix), { + const n = reduce( + pipe( + iter, + opConcatMap((a) => a.split('')) + ), + parseStream(radix), + { nodes: [root], root, stack: [{ node: root, s: '' }], parser: undefined, - }); + } + ); return n.root; } diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV4.test.ts b/packages/cspell-trie-lib/src/lib/io/importExportV4.test.ts index 4e903e25733..fe7cb03697c 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV4.test.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV4.test.ts @@ -5,7 +5,7 @@ import { describe, expect, test } from 'vitest'; import { resolveSample as resolveSamplePath } from '../../test/samples.js'; import { consolidate } from '../consolidate.js'; import * as Trie from '../index.js'; -import type { TrieNode } from '../TrieNode.js'; +import type { TrieNode } from '../TrieNode/TrieNode.js'; import * as v3 from './importExportV3.js'; import { __testing__, importTrie, serializeTrie } from './importExportV4.js'; diff --git a/packages/cspell-trie-lib/src/lib/io/importExportV4.ts b/packages/cspell-trie-lib/src/lib/io/importExportV4.ts index 1426356e7ac..86d6e6b29ce 100644 --- a/packages/cspell-trie-lib/src/lib/io/importExportV4.ts +++ b/packages/cspell-trie-lib/src/lib/io/importExportV4.ts @@ -21,13 +21,11 @@ * ``` */ -import { opConcatMap, opFilter, pipe } from '@cspell/cspell-pipe/sync'; -import type { Sequence } from 'gensequence'; -import { genSequence } from 'gensequence'; +import { opAppend, opConcatMap, opFilter, pipe, reduce } from '@cspell/cspell-pipe/sync'; -import { trieNodeToRoot } from '../trie-util.js'; -import type { TrieNode, TrieRoot } from '../TrieNode.js'; -import { FLAG_WORD } from '../TrieNode.js'; +import { trieNodeToRoot } from '../TrieNode/trie-util.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; +import { FLAG_WORD } from '../TrieNode/TrieNode.js'; import { bufferLines } from '../utils/bufferLines.js'; const EOW = '$'; // End of word @@ -93,7 +91,7 @@ export interface ExportOptions { /** * Serialize a TrieRoot. */ -export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Sequence { +export function serializeTrie(root: TrieRoot, options: ExportOptions | number = 16): Iterable { options = typeof options === 'number' ? { base: options } : options; const { base = 10, comment = '' } = options; const radix = base > 36 ? 36 : base < 10 ? 10 : base; @@ -207,7 +205,7 @@ export function serializeTrie(root: TrieRoot, options: ExportOptions | number = .replace(/.{110,130}[,]/g, '$&\n') + '\n]\n'; - return genSequence([generateHeader(radix, comment), reference]).concat(lines); + return pipe([generateHeader(radix, comment), reference], opAppend(lines)); } interface ReferenceMap { @@ -478,14 +476,19 @@ function parseStream(radix: number, iter: Iterable): TrieRoot { return parserStart({ ...acc, parser: parserStart }, s); } - genSequence(iter) - .concatMap((a) => a.split('')) - .reduce(parserMain, { + reduce( + pipe( + iter, + opConcatMap((a) => a.split('')) + ), + parserMain, + { nodes: [root], root, stack: [{ node: root, s: '' }], parser: parseReferenceIndex, - }); + } + ); return root; } diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts index 5da3f80ac59..a49fe86c84f 100644 --- a/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts +++ b/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts @@ -1,8 +1,8 @@ import { describe, expect, test } from 'vitest'; import { Trie } from '../trie.js'; -import { isWordTerminationNode } from '../trie-util.js'; -import type { TrieNode } from '../TrieNode.js'; +import { isWordTerminationNode } from '../TrieNode/trie-util.js'; +import type { TrieNode } from '../TrieNode/TrieNode.js'; import { walker } from '../walker/index.js'; import * as Sug from './suggest.js'; import type { SuggestionResultBase } from './suggestCollector.js'; diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts index cd1f47dd10f..46f1949cd78 100644 --- a/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts +++ b/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts @@ -1,5 +1,5 @@ -import { isWordTerminationNode } from '../trie-util.js'; -import type { TrieRoot } from '../TrieNode.js'; +import { isWordTerminationNode } from '../TrieNode/trie-util.js'; +import type { TrieRoot } from '../TrieNode/TrieNode.js'; import { clean } from '../utils/clean.js'; import { CompoundWordsMethod, hintedWalker, JOIN_SEPARATOR, WORD_SEPARATOR } from '../walker/index.js'; import type { GenSuggestionOptions, SuggestionOptions } from './genSuggestionsOptions.js'; diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts index bf9e9bcd2fa..7d606c10f5f 100644 --- a/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts +++ b/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts @@ -1,4 +1,4 @@ -import type { TrieNode, TrieRoot } from '../TrieNode.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; import { PairingHeap } from '../utils/PairingHeap.js'; import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from '../walker/index.js'; import type { GenSuggestionOptionsStrict, SuggestionOptions } from './genSuggestionsOptions.js'; diff --git a/packages/cspell-trie-lib/src/lib/trie.test.ts b/packages/cspell-trie-lib/src/lib/trie.test.ts index cd6554b1d18..8f20372862b 100644 --- a/packages/cspell-trie-lib/src/lib/trie.test.ts +++ b/packages/cspell-trie-lib/src/lib/trie.test.ts @@ -5,7 +5,7 @@ import { parseDictionary } from './SimpleDictionaryParser.js'; import type { SuggestionOptions } from './suggestions/genSuggestionsOptions.js'; import type { SuggestionCollectorOptions } from './suggestions/suggestCollector.js'; import { defaultTrieOptions, Trie } from './trie.js'; -import { isWordTerminationNode, orderTrie } from './trie-util.js'; +import { isWordTerminationNode, orderTrie } from './TrieNode/trie-util.js'; import { clean } from './utils/clean.js'; import { normalizeWordToLowercase } from './utils/normalizeWord.js'; diff --git a/packages/cspell-trie-lib/src/lib/trie.ts b/packages/cspell-trie-lib/src/lib/trie.ts index 7570b91db05..1bb05f8ac39 100644 --- a/packages/cspell-trie-lib/src/lib/trie.ts +++ b/packages/cspell-trie-lib/src/lib/trie.ts @@ -1,12 +1,11 @@ -import type { Sequence } from 'gensequence'; -import { genSequence } from 'gensequence'; +import { opAppend, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync'; import { CASE_INSENSITIVE_PREFIX, COMPOUND_FIX, FORBID_PREFIX, OPTIONAL_COMPOUND_FIX } from './constants.js'; -import type { FindFullResult, FindOptions, PartialFindOptions } from './find.js'; -import { createFindOptions, findLegacyCompound, findWord, findWordNode, isForbiddenWord } from './find.js'; import { genSuggestions, suggest } from './suggest.js'; import type { SuggestionCollector, SuggestionResult } from './suggestCollector.js'; import type { SuggestionOptions } from './suggestions/genSuggestionsOptions.js'; +import type { FindFullResult, FindOptions, PartialFindOptions } from './TrieNode/find.js'; +import { createFindOptions, findLegacyCompound, findWord, findWordNode, isForbiddenWord } from './TrieNode/find.js'; import { countWords, createTriFromList, @@ -14,8 +13,8 @@ import { isWordTerminationNode, iteratorTrieWords, orderTrie, -} from './trie-util.js'; -import type { PartialTrieOptions, TrieNode, TrieOptions, TrieRoot } from './TrieNode.js'; +} from './TrieNode/trie-util.js'; +import type { PartialTrieOptions, TrieNode, TrieOptions, TrieRoot } from './TrieNode/TrieNode.js'; import { clean } from './utils/clean.js'; import { mergeOptionalWithDefaults } from './utils/mergeOptionalWithDefaults.js'; import { replaceAllFactory } from './utils/util.js'; @@ -29,7 +28,7 @@ export { FORBID_PREFIX, OPTIONAL_COMPOUND_FIX, } from './constants.js'; -export { PartialTrieOptions, TrieOptions } from './TrieNode.js'; +export { PartialTrieOptions, TrieOptions } from './TrieNode/TrieNode.js'; /** @deprecated */ export const COMPOUND = COMPOUND_FIX; @@ -156,13 +155,15 @@ export class Trie { /** * Provides an ordered sequence of words with the prefix of text. */ - completeWord(text: string): Sequence { + completeWord(text: string): Iterable { const n = this.find(text); const compoundChar = this.options.compoundCharacter; - const subNodes = iteratorTrieWords(n || {}) - .filter((w) => w[w.length - 1] !== compoundChar) - .map((suffix) => text + suffix); - return genSequence(n && isWordTerminationNode(n) ? [text] : []).concat(subNodes); + const subNodes = pipe( + iteratorTrieWords(n || {}), + opFilter((w) => w[w.length - 1] !== compoundChar), + opMap((suffix) => text + suffix) + ); + return pipe(n && isWordTerminationNode(n) ? [text] : [], opAppend(subNodes)); } /** @@ -210,7 +211,7 @@ export class Trie { /** * Returns an iterator that can be used to get all words in the trie. For some dictionaries, this can result in millions of words. */ - words(): Sequence { + words(): Iterable { return iteratorTrieWords(this.root); } diff --git a/packages/cspell-trie-lib/src/lib/utils/mergeOptionalWithDefaults.ts b/packages/cspell-trie-lib/src/lib/utils/mergeOptionalWithDefaults.ts index cc04b85eb0f..a7c6715aedf 100644 --- a/packages/cspell-trie-lib/src/lib/utils/mergeOptionalWithDefaults.ts +++ b/packages/cspell-trie-lib/src/lib/utils/mergeOptionalWithDefaults.ts @@ -1,5 +1,5 @@ import { defaultTrieOptions } from '../constants.js'; -import type { PartialTrieOptions, TrieOptions } from '../TrieNode.js'; +import type { PartialTrieOptions, TrieOptions } from '../TrieNode/TrieNode.js'; import { mergeDefaults } from './mergeDefaults.js'; export function mergeOptionalWithDefaults(options: PartialTrieOptions): TrieOptions { diff --git a/packages/cspell-trie-lib/src/lib/walker/hintedWalker.test.ts b/packages/cspell-trie-lib/src/lib/walker/hintedWalker.test.ts index 31346552209..d1b50ac9a97 100644 --- a/packages/cspell-trie-lib/src/lib/walker/hintedWalker.test.ts +++ b/packages/cspell-trie-lib/src/lib/walker/hintedWalker.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { parseLinesToDictionary } from '../SimpleDictionaryParser.js'; -import { createTriFromList, orderTrie } from '../trie-util.js'; +import { createTriFromList, orderTrie } from '../TrieNode/trie-util.js'; import type { HintedWalkerIterator } from './hintedWalker.js'; import { hintedWalker } from './hintedWalker.js'; import type { YieldResult } from './walkerTypes.js'; diff --git a/packages/cspell-trie-lib/src/lib/walker/hintedWalker.ts b/packages/cspell-trie-lib/src/lib/walker/hintedWalker.ts index 1e83ba41be9..bf0291708fb 100644 --- a/packages/cspell-trie-lib/src/lib/walker/hintedWalker.ts +++ b/packages/cspell-trie-lib/src/lib/walker/hintedWalker.ts @@ -1,4 +1,4 @@ -import type { TrieNode, TrieRoot } from '../TrieNode.js'; +import type { TrieNode, TrieRoot } from '../TrieNode/TrieNode.js'; import { isDefined } from '../utils/isDefined.js'; import type { YieldResult } from './walkerTypes.js'; import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from './walkerTypes.js'; diff --git a/packages/cspell-trie-lib/src/lib/walker/walker.test.ts b/packages/cspell-trie-lib/src/lib/walker/walker.test.ts index 8b88ad8cba9..a17e0783f92 100644 --- a/packages/cspell-trie-lib/src/lib/walker/walker.test.ts +++ b/packages/cspell-trie-lib/src/lib/walker/walker.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { createTriFromList, orderTrie } from '../trie-util.js'; +import { createTriFromList, orderTrie } from '../TrieNode/trie-util.js'; import { walker } from './walker.js'; import type { WalkerIterator, YieldResult } from './walkerTypes.js'; diff --git a/packages/cspell-trie-lib/src/lib/walker/walker.ts b/packages/cspell-trie-lib/src/lib/walker/walker.ts index 3b67a075c46..3e5c5596591 100644 --- a/packages/cspell-trie-lib/src/lib/walker/walker.ts +++ b/packages/cspell-trie-lib/src/lib/walker/walker.ts @@ -1,4 +1,4 @@ -import type { TrieNode } from '../TrieNode.js'; +import type { TrieNode } from '../TrieNode/TrieNode.js'; import type { WalkerIterator } from './walkerTypes.js'; import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from './walkerTypes.js'; diff --git a/packages/cspell-trie-lib/src/lib/walker/walkerTypes.ts b/packages/cspell-trie-lib/src/lib/walker/walkerTypes.ts index 4a22e7a58ca..491db5a01ce 100644 --- a/packages/cspell-trie-lib/src/lib/walker/walkerTypes.ts +++ b/packages/cspell-trie-lib/src/lib/walker/walkerTypes.ts @@ -1,4 +1,4 @@ -import type { TrieNode } from '../TrieNode.js'; +import type { TrieNode } from '../TrieNode/TrieNode.js'; export const JOIN_SEPARATOR = '+'; export const WORD_SEPARATOR = ' '; diff --git a/packages/cspell-trie-lib/src/test/util.test.helper.ts b/packages/cspell-trie-lib/src/test/util.test.helper.ts index 43e9b0402e9..17c91cd2012 100644 --- a/packages/cspell-trie-lib/src/test/util.test.helper.ts +++ b/packages/cspell-trie-lib/src/test/util.test.helper.ts @@ -1,4 +1,4 @@ -import type { TrieNode } from '../lib/TrieNode.js'; +import type { TrieNode } from '../lib/TrieNode/TrieNode.js'; export function emitTrieWords(trie: TrieNode) { let count = 0; diff --git a/packages/cspell-trie/src/app.ts b/packages/cspell-trie/src/app.ts index fbe44ea5ba9..be90e5db41a 100644 --- a/packages/cspell-trie/src/app.ts +++ b/packages/cspell-trie/src/app.ts @@ -74,7 +74,7 @@ export async function run(program: commander.Command, argv: string[]): Promise = Trie.iteratorTrieWords(root); + const words: Sequence = genSequence(Trie.iteratorTrieWords(root)); const outputStream = await pOutputStream; return new Promise((resolve) => { stream.Readable.from(words.map((a) => a + '\n'))