improve logging (#149)

packages/cspell-tools/.gitignore

@@ -42,4 +42,7 @@ jspm_packages
 # Do not include test or map files in the dist.
 dist/**
 
-temp
+temp
+
+/*.trie.gz
+/*.trie

packages/cspell-tools/src/app.ts

@@ -4,7 +4,8 @@
 // # !/usr/bin/env node --max_old_space_size=8192
 
 
-import { compileWordList, compileTrie } from './compiler';
+import { compileWordList, compileTrie, Logger } from './compiler';
+import * as compiler from './compiler';
 import * as path from 'path';
 import * as program from 'commander';
 import * as glob from 'glob';
@@ -37,6 +38,12 @@ interface CompileTrieOptions extends CompileCommonOptions {
     trie3: boolean;
 }
 
+const log: Logger = (message?: any, ...optionalParams: any[]) => {
+    console.log(`${new Date().toISOString()} ${message}`, ...optionalParams);
+};
+
+compiler.setLogger(log);
+
 export function run(
     program: program.Command,
     argv: string[]
@@ -115,10 +122,12 @@ async function processAction(
     const globResults = await Promise.all(src.map(s => globP(s)));
     const filesToProcess = genSequence(globResults)
         .concatMap(files => files)
-        .map(async s => {
-            const words = await streamWordsFromFile(s, readerOptions);
+        .map(async filename => {
+            log(`Reading ${path.basename(filename)}`);
+            const words = await streamWordsFromFile(filename, readerOptions);
+            log(`Done reading ${path.basename(filename)}`);
             const f: FileToProcess = {
-                src: s,
+                src: filename,
                 words,
             };
             return f;
@@ -128,7 +137,7 @@ async function processAction(
     ? processFiles(action, filesToProcess, toMergeTargetFile(options.merge, options.output, ext))
     : processFilesIndividually(action, filesToProcess, s => toTargetFile(s, options.output, ext));
     await r;
-    console.log(`Complete.`);
+    log(`Complete.`);
 }
 
 function toFilename(name: string, ext: string) {
@@ -155,9 +164,9 @@ async function processFilesIndividually(
     .map(async pFtp => {
         const { src, words } = await pFtp;
         const dst = srcToTarget(src);
-        console.log('Process "%s" to "%s"', src, dst);
+        log('Process "%s" to "%s"', src, dst);
         await action(words, dst);
-        console.log('Done "%s" to "%s"', src, dst);
+        log('Done "%s" to "%s"', src, dst);
     });
 
     for (const p of toProcess) {
@@ -176,12 +185,12 @@ async function processFiles(
     const words = genSequence(toProcess)
     .map(ftp => {
         const { src } = ftp;
-        console.log('Process "%s" to "%s"', src, dst);
+        log('Process "%s" to "%s"', src, dst);
         return ftp;
     })
     .concatMap( ftp => ftp.words );
     await action(words, dst);
-    console.log('Done "%s"', dst);
+    log('Done "%s"', dst);
 }
 
 if (require.main === module) {

packages/cspell-tools/src/compiler/wordListCompiler.ts

@@ -11,6 +11,18 @@ const regNonWordOrSpace = XRegExp("[^\\p{L}' ]+", 'gi');
 const regExpSpaceOrDash = /(?:\s+)|(?:-+)/g;
 const regExpRepeatChars = /(.)\1{3,}/i;
 
+export type Logger = (message?: any, ...optionalParams: any[]) => void;
+
+let log: Logger = defaultLogger;
+
+export function setLogger(logger?: Logger) {
+    log = logger ?? defaultLogger;
+}
+
+function defaultLogger(message?: any, ...optionalParams: any[]) {
+    console.log(message, ...optionalParams);
+}
+
 export function normalizeWords(lines: Sequence<string>) {
     return lines.concatMap(line => lineToWords(line));
 }
@@ -85,12 +97,17 @@ export interface CompileTrieOptions {
 }
 
 export async function compileTrie(words: Sequence<string>, destFilename: string, options: CompileTrieOptions): Promise<void> {
+    log('Reading Words into Trie');
     const base = options.base ?? 32;
     const version = options.trie3 ? 3 : 1;
     const destDir = path.dirname(destFilename);
     const pDir = mkdirp(destDir);
     const pRoot = normalizeWordsToTrie(words);
     const [root] = await Promise.all([pRoot, pDir]);
+    log('Reduce duplicate word endings');
+    const trie = Trie.consolidate(root);
 
-    return writeSeqToFile(Trie.serializeTrie(root, { base, comment: 'Built by cspell-tools.', version }), destFilename);
+    log(`Writing to file ${path.basename(destFilename)}`);
+    await writeSeqToFile(Trie.serializeTrie(trie, { base, comment: 'Built by cspell-tools.', version }), destFilename);
+    log(`Done writing to file ${path.basename(destFilename)}`);
 }

packages/cspell-trie-lib/src/lib/TrieBuilder.test.ts

@@ -5,8 +5,8 @@ describe('Validate TrieBuilder', () => {
     test('builder', () => {
         const builder = new TrieBuilder();
         builder.insert(sampleWords);
-        const trie = builder.build();
-        expect([...trie.words()]).toEqual(sampleWords.sort());
+        const trie = builder.build(true);
+        expect([...trie.words()].sort()).toEqual(sampleWords.sort());
         expect(countNodes(trie.root)).toBe(113);
     });
 

packages/cspell-trie-lib/src/lib/TrieBuilder.ts

@@ -136,10 +136,10 @@ export class TrieBuilder {
         this.signatures.clear();
     }
 
-    build(): Trie {
+    build(consolidateSuffixes: boolean = false): Trie {
         const root = this._root;
         // Reset the builder to prevent updating the trie in the background.
         this.reset();
-        return new Trie(consolidate(root));
+        return new Trie(consolidateSuffixes ? consolidate(root) : root);
     }
 }

packages/cspell-trie-lib/src/lib/importExportV3.test.ts

@@ -23,15 +23,15 @@ describe('Import/Export', () => {
 
     test('tests serialize / deserialize specialCharacters', async () => {
         const trie = Trie.buildTrie(specialCharacters).root;
-        const data = [...serializeTrie(trie, 10)];
+        const data = [...serializeTrie(consolidate(trie), 10)];
         const root = importTrie(data);
         const words = [...Trie.iteratorTrieWords(root)];
         expect(words).toEqual([...specialCharacters].sort());
     });
 
     test('tests serialize / deserialize', async () => {
         const trie = Trie.buildTrie(sampleWords).root;
-        const data = [...serializeTrie(trie, { base: 10, comment: 'Sample Words' })].join('');
+        const data = [...serializeTrie(consolidate(trie), { base: 10, comment: 'Sample Words' })].join('');
         const root = importTrie(data.split('\n').map(a => a ? a + '\n' : a));
         const words = [...Trie.iteratorTrieWords(root)];
         expect(words).toEqual([...sampleWords].sort());

packages/cspell-trie-lib/src/lib/index.ts

@@ -4,4 +4,5 @@ export * from './util';
 export * from './walker';
 export * from './importExport';
 export * from './TrieBuilder';
+export * from './consolidate';
 export {SuggestionResult, MaxCost, suggestionCollector, SuggestionCollector, CompoundWordsMethod} from './suggest';