From 9b3e6702d9f2bdd3c2dd8b60cc089f130e20d393 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Thu, 16 May 2019 23:04:47 +0200 Subject: [PATCH] Move away from rxjs to iterables. --- .travis.yml | 1 - appveyor.yml | 1 - package-lock.json | 6 +- package.json | 2 +- packages/cspell-glob/package.json | 2 +- packages/cspell-lib/.vscode/launch.json | 2 +- packages/cspell-lib/CHANGELOG.md | 3 + packages/cspell-lib/package-lock.json | 54 +++++++- packages/cspell-lib/package.json | 13 +- .../cspell-lib/src/async/asyncIterable.ts | 13 ++ .../cspell-lib/src/file/fileReader.test.ts | 107 ++++++++------- packages/cspell-lib/src/file/fileReader.ts | 124 +++++++++++++----- .../cspell-lib/src/file/fileWriter.test.ts | 84 +++++------- packages/cspell-lib/src/file/fileWriter.ts | 20 +-- packages/cspell-lib/src/index.ts | 1 + packages/cspell-tools/package-lock.json | 16 +-- packages/cspell-tools/package.json | 8 +- .../src/compiler/fileReader.test.ts | 58 -------- .../cspell-tools/src/compiler/fileReader.ts | 1 - .../src/compiler/fileWriter.test.ts | 22 +--- .../cspell-tools/src/compiler/fileWriter.ts | 11 +- .../src/compiler/wordListCompiler.test.ts | 25 ++-- .../src/compiler/wordListCompiler.ts | 62 ++++----- packages/cspell-trie-lib/package-lock.json | 2 +- packages/cspell-trie-lib/package.json | 4 +- packages/cspell-trie/.vscode/launch.json | 2 +- packages/cspell-trie/package-lock.json | 16 +-- packages/cspell-trie/package.json | 6 +- packages/cspell-trie/src/index.test.ts | 14 ++ packages/cspell/CHANGELOG.md | 3 + packages/cspell/package-lock.json | 89 ++++++++++++- packages/cspell/package.json | 8 +- .../SpellingDictionary/DictionaryLoader.ts | 35 +++-- .../SpellingDictionary.test.ts | 55 +------- .../SpellingDictionary/SpellingDictionary.ts | 37 ++---- .../SpellingDictionaryCollection.test.ts | 82 ++++++------ packages/cspell/src/index.ts | 1 - packages/cspell/src/textValidator.test.ts | 42 +++--- packages/cspell/src/util/fileReader.test.ts | 3 +- packages/cspell/src/util/fileReader.ts | 14 +- .../src/util/iterableIteratorLib.test.ts | 19 +++ .../cspell/src/util/iterableIteratorLib.ts | 11 ++ packages/cspell/src/wordListHelper.test.ts | 63 ++++----- packages/cspell/src/wordListHelper.ts | 33 +++-- 44 files changed, 608 insertions(+), 567 deletions(-) create mode 100644 packages/cspell-lib/src/async/asyncIterable.ts delete mode 100644 packages/cspell-tools/src/compiler/fileReader.test.ts delete mode 100644 packages/cspell-tools/src/compiler/fileReader.ts create mode 100644 packages/cspell-trie/src/index.test.ts create mode 100644 packages/cspell/src/util/iterableIteratorLib.test.ts create mode 100644 packages/cspell/src/util/iterableIteratorLib.ts diff --git a/.travis.yml b/.travis.yml index f540135c4fd..3afa84f2970 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ node_js: - "node" - "12" - "10" - - "8" install: # the prepare script run during install currently does clean-build diff --git a/appveyor.yml b/appveyor.yml index 73c078f067a..87529a6f7e1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,7 +9,6 @@ environment: # - nodejs_version: "12" - nodejs_version: "Current" - nodejs_version: "10" - - nodejs_version: "8" # Install scripts. (runs after repo cloning) install: diff --git a/package-lock.json b/package-lock.json index c5d9e088021..8a20d643ee4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1066,9 +1066,9 @@ } }, "@types/node": { - "version": "8.10.48", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.48.tgz", - "integrity": "sha512-c35YEBTkL4rzXY2ucpSKy+UYHjUBIIkuJbWYbsGIrKLEWU5dgJMmLkkIb3qeC3O3Tpb1ZQCwecscvJTDjDjkRw==", + "version": "10.14.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.6.tgz", + "integrity": "sha512-Fvm24+u85lGmV4hT5G++aht2C5I4Z4dYlWZIh62FAfFO/TfzXtPpoLI6I7AuBWkIFqZCnhFOoTT7RjjaIL5Fjg==", "dev": true }, "@types/xregexp": { diff --git a/package.json b/package.json index e952908ccbb..804ca80e070 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "@types/minimatch": "^3.0.3", "@types/mocha": "^5.2.6", "@types/mock-require": "^2.0.0", - "@types/node": "^8.10.48", + "@types/node": "^10.14.6", "@types/xregexp": "^3.0.29", "chai": "^4.2.0", "coveralls": "^3.0.3", diff --git a/packages/cspell-glob/package.json b/packages/cspell-glob/package.json index fecd18aedc0..9b079b08550 100644 --- a/packages/cspell-glob/package.json +++ b/packages/cspell-glob/package.json @@ -56,7 +56,7 @@ ] }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" }, "dependencies": { "micromatch": "^4.0.2" diff --git a/packages/cspell-lib/.vscode/launch.json b/packages/cspell-lib/.vscode/launch.json index 2ed53258a27..f7215376b09 100644 --- a/packages/cspell-lib/.vscode/launch.json +++ b/packages/cspell-lib/.vscode/launch.json @@ -18,7 +18,7 @@ "type": "node", "request": "launch", "name": "Run Mocha Current File", - "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", + "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha", "runtimeArgs": [ ], "args": [ diff --git a/packages/cspell-lib/CHANGELOG.md b/packages/cspell-lib/CHANGELOG.md index 8d87b80e9b0..5a376581ec7 100644 --- a/packages/cspell-lib/CHANGELOG.md +++ b/packages/cspell-lib/CHANGELOG.md @@ -1,5 +1,8 @@ # Change log +## [4.0.0] +* **Breaking Changes** drops dependency upon rxjs and moves to [AsyncIterator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/asyncIterator) + ## [3.05] * Update dependencies and use `rxjs-stream` for reading files. diff --git a/packages/cspell-lib/package-lock.json b/packages/cspell-lib/package-lock.json index d980c746694..74a41356083 100644 --- a/packages/cspell-lib/package-lock.json +++ b/packages/cspell-lib/package-lock.json @@ -1,9 +1,17 @@ { "name": "cspell-lib", - "version": "3.0.8", + "version": "4.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "requires": { + "once": "^1.4.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -12,15 +20,51 @@ "safer-buffer": ">= 2.1.2 < 3" } }, - "rxjs-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/rxjs-stream/-/rxjs-stream-3.0.1.tgz", - "integrity": "sha512-KZ+B7MWw2mygAsLJJ+mjYkCXHtSeYRXWhUVQBQRX4NgBmEiAEGAmVz7epY+8Gj0REL22W0T8ZbysFbzI/76ikA==" + "iterable-to-stream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-1.0.1.tgz", + "integrity": "sha512-O62gD5ADMUGtJoOoM9U6LQ7i4byPXUNoHJ6mqsmkQJcom331ZJGDApWgDESWyBMEHEJRjtHozgIiTzYo9RU4UA==" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=" + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "stream-to-iterator": { + "version": "3.0.2-0", + "resolved": "https://registry.npmjs.org/stream-to-iterator/-/stream-to-iterator-3.0.2-0.tgz", + "integrity": "sha512-+HZedhN1PE/9Q6aH2bM0c75+s37/mDdodmgqVp9G5f/lyuo2d7VFbmQykxyxtrRPsxpeha0yPyjYozYrN8YzRQ==", + "requires": { + "p-defer": "1.0.0", + "pump": "^3.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" } } } diff --git a/packages/cspell-lib/package.json b/packages/cspell-lib/package.json index a46c1c2ee8b..7b14b3f75d6 100644 --- a/packages/cspell-lib/package.json +++ b/packages/cspell-lib/package.json @@ -1,6 +1,6 @@ { "name": "cspell-lib", - "version": "3.0.8", + "version": "4.0.0", "description": "A library of useful functions used across various cspell tools.", "main": "dist/index.js", "typings": "dist/index.d.ts", @@ -16,7 +16,7 @@ "clean": "rimraf dist", "clean-build": "npm run clean && npm run build", "coverage": "NODE_ENV=test nyc --silent --no-clean --temp-dir=../../.nyc_output npm run test-ts", - "test-ts": "rimraf temp ; NODE_ENV=test mocha --require ts-node/register --recursive --bail \"src/**/*.test.ts\"", + "test-ts": "../../node_modules/.bin/rimraf temp ; NODE_ENV=test ../../node_modules/.bin/mocha --require ts-node/register --recursive \"src/**/*.test.ts\"", "test-watch": "../../node_modules/.bin/mocha --require ts-node/register --watch --recursive \"src/**/*.test.ts\"", "test": "rimraf temp ; mocha --recursive ./dist/**/*.test.js" }, @@ -35,11 +35,10 @@ "homepage": "https://github.com/streetsidesoftware/cspell#readme", "dependencies": { "iconv-lite": "^0.4.24", - "rxjs-stream": "^3.0.1" - }, - "peerDependencies": { - "rxjs": "^6.0.0" + "iterable-to-stream": "^1.0.1", + "stream-to-iterator": "^3.0.2-0" }, + "peerDependencies": {}, "nyc": { "include": [ "src/**/*.ts" @@ -59,6 +58,6 @@ ] }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" } } diff --git a/packages/cspell-lib/src/async/asyncIterable.ts b/packages/cspell-lib/src/async/asyncIterable.ts new file mode 100644 index 00000000000..2b83536768f --- /dev/null +++ b/packages/cspell-lib/src/async/asyncIterable.ts @@ -0,0 +1,13 @@ + +/** + * Reads an entire iterable and converts it into a promise. + * @param asyncIterable the async iterable to wait for. + */ +export async function toArray(asyncIterable: AsyncIterable | Iterable | Iterable>): Promise { + const data: T[] = []; + for await (const item of asyncIterable) { + data.push(item); + } + return data; +} + diff --git a/packages/cspell-lib/src/file/fileReader.test.ts b/packages/cspell-lib/src/file/fileReader.test.ts index 1b67908299c..e3cbf84dba7 100644 --- a/packages/cspell-lib/src/file/fileReader.test.ts +++ b/packages/cspell-lib/src/file/fileReader.test.ts @@ -1,9 +1,9 @@ import { expect } from 'chai'; import * as fReader from './fileReader'; -import { of } from 'rxjs'; -import { toArray } from 'rxjs/operators'; import * as fs from 'fs-extra'; import * as path from 'path'; +import { Readable } from 'stream'; +import * as asyncIterable from '../async/asyncIterable'; describe('Validate the fileReader', () => { const samplePath = path.join(__dirname, '..', '..', 'samples'); @@ -14,82 +14,79 @@ describe('Validate the fileReader', () => { 'cities.noEOL.txt', ].map(f => path.join(samplePath, f)); - it('tests stringsToLines', () => { - const strings = of('a1\n2\n3\n4', '5\n6'); - return fReader.stringsToLinesRx(strings).pipe(toArray()) - .toPromise().then((a) => { - expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6']); - }); + it('tests reading a file', async () => { + const expected = await fs.readFile(__filename, 'utf8'); + const result = await fReader.readFile(__filename, 'utf8'); + expect(result).to.be.equal(expected); }); - it('tests stringsToLines trailing new line', () => { - const strings = of('a1\n2\n3\n4', '5\n6\n'); - return fReader.stringsToLinesRx(strings).pipe(toArray()).toPromise().then((a) => { - expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6', '']); - }); + it('tests stringsToLines', async () => { + const strings = stringToStream('a1\n2\n3\n4', '5\n6'); + const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); + expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6']); }); - it('test the file reader', () => { - return fReader.stringsToLinesRx(fReader.textFileStreamRx(__filename)) - .pipe(toArray()) - .toPromise() - .then(lines => { - const actual = lines.join('\n'); - const expected = fs.readFileSync(__filename, 'UTF-8'); - expect(actual).to.equal(expected); - }); + it('tests stringsToLines trailing new line', async () => { + const strings = stringToStream('a1\n2\n3\n4', '5\n6\n'); + const a = await asyncIterable.toArray(fReader.streamLineByLineAsync(strings)); + expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6', '']); }); - it('test the lineReaderRx', () => { - return fReader.lineReaderRx(__filename) - .pipe(toArray()) - .toPromise() - .then(lines => { - const expected = fs.readFileSync(__filename, 'UTF-8').split('\n'); - expect(lines).to.deep.equal(expected); - }); + it('test the file reader', async () => { + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); + const actual = lines.join('\n'); + const expected = fs.readFileSync(__filename, 'UTF-8'); + expect(actual).to.equal(expected); + }); + + it('test the lineReaderAsync', async () => { + const lines = await asyncIterable.toArray(fReader.lineReaderAsync(__filename)); + const expected = fs.readFileSync(__filename, 'UTF-8').split('\n'); + expect(lines).to.deep.equal(expected); }); it('tests reading the cities sample', async () => { - const lines = await fReader.lineReaderRx(fileCities) - .pipe(toArray()) - .toPromise(); + const lines = await asyncIterable.toArray(fReader.lineReaderAsync(fileCities)); const file = await fs.readFile(fileCities, 'utf8'); expect(lines).to.be.deep.equal(file.split('\n')); }); - it('tests streamFileLineByLineRx', async () => { + it('tests streamFileLineByLineAsync', async () => { await Promise.all(sampleFiles .map(async filename => { - const lines = await fReader.streamFileLineByLineRx(filename) - .pipe(toArray()) - .toPromise(); + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(filename)); const file = await fs.readFile(filename, 'utf8'); expect(lines, `compare to file: ${filename}`).to.be.deep.equal(file.split(/\r?\n/)); })); }); - it('tests streamFileLineByLineRx 2', async () => { - const lines = await fReader.streamFileLineByLineRx(__filename) - .pipe(toArray()) - .toPromise(); + it('tests streamFileLineByLineAsync 2', async () => { + const lines = await asyncIterable.toArray(fReader.streamFileLineByLineAsync(__filename)); const file = await fs.readFile(__filename, 'utf8'); expect(lines).to.be.deep.equal(file.split('\n')); }); - it('test missing file', () => { - return fReader.lineReaderRx(__filename + 'not.found') - .pipe(toArray()) - .toPromise() - .then( - () => { - expect('not to be here').to.be.true; - }, - (e) => { - expect(e).to.be.instanceof(Error); - expect(e.code).to.be.equal('ENOENT'); - } - ); + it('test missing file', async () => { + const result = asyncIterable.toArray(fReader.lineReaderAsync(__filename + 'not.found')); + return result.then( + () => { + expect('not to be here').to.be.true; + }, + (e) => { + expect(e).to.be.instanceof(Error); + expect(e.code).to.be.equal('ENOENT'); + } + ); }); - }); + +function stringToStream(...strings: string[]): NodeJS.ReadableStream { + return new Readable({ + read: function() { + for (const s of strings) { + this.push(s); + } + this.push(null); + } + }); +} diff --git a/packages/cspell-lib/src/file/fileReader.ts b/packages/cspell-lib/src/file/fileReader.ts index cdaef5c40ba..0a08d7a39ba 100644 --- a/packages/cspell-lib/src/file/fileReader.ts +++ b/packages/cspell-lib/src/file/fileReader.ts @@ -1,23 +1,36 @@ // cSpell:ignore curr // cSpell:words zlib iconv -// cSpell:enableCompoundWords import * as fs from 'fs'; -import {Observable, Subject, concat, of} from 'rxjs'; -import {scan, concatMap, merge} from 'rxjs/operators'; import * as iconv from 'iconv-lite'; import * as zlib from 'zlib'; import * as readline from 'readline'; -import { streamToStringRx } from 'rxjs-stream'; +import {} from 'stream'; -const defaultEncoding = 'utf8'; +const defaultEncoding: BufferEncoding = 'utf8'; + +export function readFile(filename: string, encoding: BufferEncoding = defaultEncoding): Promise { + return new Promise((resolve, reject) => { + const data: string[] = []; + const stream = prepareFileStream(filename, encoding, reject); + let resolved = false; + function complete() { + resolve(data.join('')); + resolved = resolved || (resolve(data.join('')), true); + } + stream.on('error', reject); + stream.on('data', (d: string) => data.push(d)); + stream.on('close', complete); + stream.on('end', complete); + }); +} /** * Reads a file line by line. The last value emitted by the Observable is always an empty string. * @param filename * @param encoding defaults to 'utf8' */ -export function lineReaderRx(filename: string, encoding: string = defaultEncoding): Observable { - return stringsToLinesRx(textFileStreamRx(filename, encoding)); +export function lineReaderAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterable { + return streamFileLineByLineAsync(filename, encoding); } function prepareFileStream( @@ -36,12 +49,26 @@ function prepareFileStream( return stream; } -export function textFileStreamRx(filename: string, encoding: string = defaultEncoding): Observable { - const errorHandler = new Subject(); - const fnError = (e: Error) => errorHandler.error(e); +/** + * Emit a file line by line + * @param filename full path to the file to read. + * @param encoding defaults to 'utf8' + */ +export function streamFileLineByLineAsync(filename: string, encoding: BufferEncoding = defaultEncoding): AsyncIterableIterator { + const fnError = (e: Error) => { + iter.throw && iter.throw(e); + }; const stream = prepareFileStream(filename, encoding, fnError); - stream.on('end', () => errorHandler.complete()); - return streamToStringRx(stream, encoding).pipe(merge(errorHandler)); + const iter = streamLineByLineAsync(stream); + return iter; +} + +type Resolve = (value?: T | PromiseLike) => void; +type Reject = (reason?: any) => void; + +interface Resolvers> { + resolve: Resolve; + reject: Reject; } /** @@ -49,38 +76,75 @@ export function textFileStreamRx(filename: string, encoding: string = defaultEnc * @param filename full path to the file to read. * @param encoding defaults to 'utf8' */ -export function streamFileLineByLineRx(filename: string, encoding: string = defaultEncoding): Observable { - const subject = new Subject(); +export function streamLineByLineAsync(stream: NodeJS.ReadableStream, encoding: BufferEncoding = defaultEncoding): AsyncIterableIterator { let data = '.'; - const fnError = (e: Error) => subject.error(e); + let done = false; + let error: Error | any; + const buffer: string[] = []; + const pending: Resolvers[] = []; + const fnError = (e: Error | any) => { error = e; }; const fnComplete = () => { // readline will consume the last newline without emitting an empty last line. // If the last data read contains a new line, then emit an empty string. if (data.match(/(?:(?:\r?\n)|(?:\r))$/)) { - subject.next(''); + buffer.push(''); } - subject.complete(); + processBuffer(); + done = true; }; - const stream = prepareFileStream(filename, encoding, fnError); // We want to capture the last line. - stream.on('data', d => data = d); + stream.on('data', d => data = dataToString(d, encoding)); stream.on('error', fnError); const rl = readline.createInterface({ input: stream, terminal: false, }); rl.on('close', fnComplete); - rl.on('line', (text: string) => subject.next(text)); - return subject; + rl.on('line', (text: string) => { + buffer.push(text); + processBuffer(); + }); + + function registerPromise(resolve: Resolve>, reject: Reject) { + pending.push({ resolve, reject }); + processBuffer(); + } + + function processBuffer() { + if (error && pending.length && !buffer.length) { + const p = pending.shift()!; + p.reject(error); + return; + } + while (pending.length && buffer.length) { + const p = pending.shift()!; + const b = buffer.shift()!; + p.resolve({ done: false, value: b }); + } + if (!done) { + pending.length ? rl.resume() : rl.pause(); + } + if (done && pending.length && !buffer.length) { + const p = pending.shift()!; + p.resolve({ done } as IteratorResult) + } + } + + const iter: AsyncIterableIterator = { + [Symbol.asyncIterator]: () => iter, + next() { return new Promise(registerPromise); }, + throw(e?: any) { + fnError(e); + return new Promise(registerPromise); + }, + }; + + return iter; } -export function stringsToLinesRx(strings: Observable): Observable { - return concat(strings, of('\n')).pipe( - scan((last: { lines: string[], remainder: string }, curr: string) => { - const parts = (last.remainder + curr).split(/\r?\n/); - const lines = parts.slice(0, -1); - const remainder = parts.slice(-1)[0]; - return {lines, remainder}; - }, { lines: [], remainder: ''}), - concatMap(emit => emit.lines)); +function dataToString(data: string | Buffer, encoding: BufferEncoding = 'utf8'): string { + if (typeof data === 'string') { + return data; + } + return data.toString(encoding); } diff --git a/packages/cspell-lib/src/file/fileWriter.test.ts b/packages/cspell-lib/src/file/fileWriter.test.ts index 011df050897..2584a47f12b 100644 --- a/packages/cspell-lib/src/file/fileWriter.test.ts +++ b/packages/cspell-lib/src/file/fileWriter.test.ts @@ -1,83 +1,61 @@ import { expect } from 'chai'; import * as fileWriter from './fileWriter'; -import * as fileReader from './fileReader'; -import { fromEvent, from } from 'rxjs'; -import { flatMap, concatMap, reduce, take, map } from 'rxjs/operators'; import * as loremIpsum from 'lorem-ipsum'; import * as path from 'path'; import { mkdirp } from 'fs-extra'; -import * as fse from 'fs-extra'; +import { readFile } from './fileReader'; describe('Validate the writer', () => { - it('tests writing an Observable and reading it back.', () => { + it('tests writing data and reading it back.', async () => { // cspell:ignore éåáí const text = loremIpsum({ count: 1000, format: 'plain', units: 'words'}) + ' éåáí'; const data = text.split(/\b/); const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing-an-observable.txt'); - return from(mkdirp(path.dirname(filename))).pipe( - flatMap(() => { - return fileWriter.writeToFileRxP(filename, from(data)); - }), - concatMap(() => fileReader.textFileStreamRx(filename)), - reduce((a, b) => a + b), - ).toPromise() - .then(result => { - expect(result).to.equal(text); - }); + await mkdirp(path.dirname(filename)); + await fileWriter.writeToFileIterableP(filename, data); + const result = await readFile(filename, 'utf8'); + expect(result).to.equal(text); }); - it('tests writing an Observable and reading it back. gz', () => { + it('tests writing data and reading it back. gz', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words'}) + ' éåáí'; const data = text.split(/\b/); const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing-an-observable.txt.gz'); - return from(mkdirp(path.dirname(filename))).pipe( - flatMap(() => { - return fileWriter.writeToFileRxP(filename, from(data)); - }), - concatMap(() => fileReader.textFileStreamRx(filename)), - reduce((a, b) => a + b), - ).toPromise() - .then(result => { - expect(result).to.equal(text); - }); + await mkdirp(path.dirname(filename)); + await fileWriter.writeToFileIterableP(filename, data); + const result = await readFile(filename, 'utf8'); + expect(result).to.equal(text); }); - it('tests writeToFile', () => { + it('tests writeToFile', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words'}) + ' éåáí'; const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing.txt'); - return from(mkdirp(path.dirname(filename))).pipe( - flatMap(() => { - const wStream = fileWriter.writeToFile(filename, text); - return fromEvent(wStream, 'close'); - }), - take(1), - concatMap(() => fse.readFile(filename)), - map(buffer => buffer.toString('utf8')), - take(1), - ).toPromise() - .then(result => { - expect(result).to.equal(text); - }); + await mkdirp(path.dirname(filename)); + const wStream = fileWriter.writeToFile(filename, text); + await new Promise((resolve, reject) => { + wStream.on('close', resolve); + wStream.on('error', reject); + }); + + const result = await readFile(filename, 'utf8'); + expect(result).to.equal(text); }); - it('tests writeToFile zip', () => { + it('tests writeToFile zip', async () => { const text = loremIpsum({ count: 1000, format: 'plain', units: 'words'}) + ' éåáí'; const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing.txt.gz'); - return from(mkdirp(path.dirname(filename))).pipe( - flatMap(() => { - const wStream = fileWriter.writeToFile(filename, text); - return fromEvent(wStream, 'close'); - }), - take(1), - concatMap(() => fileReader.textFileStreamRx(filename)), - reduce((a, b) => a + b), - ).toPromise() - .then(result => { - expect(result).to.equal(text); - }); + await mkdirp(path.dirname(filename)); + const wStream = fileWriter.writeToFile(filename, text); + await new Promise((resolve, reject) => { + wStream.on('close', resolve); + wStream.on('error', reject); + }); + + const result = await readFile(filename, 'utf8'); + expect(result).to.equal(text); }); }); diff --git a/packages/cspell-lib/src/file/fileWriter.ts b/packages/cspell-lib/src/file/fileWriter.ts index 5d2e108993c..92ab10925e8 100644 --- a/packages/cspell-lib/src/file/fileWriter.ts +++ b/packages/cspell-lib/src/file/fileWriter.ts @@ -2,29 +2,21 @@ import * as fs from 'fs'; import * as zlib from 'zlib'; import * as stream from 'stream'; -import {Observable} from 'rxjs'; -import { rxToStream } from 'rxjs-stream'; +import { iterableToStream } from 'iterable-to-stream'; export function writeToFile(filename: string, data: string) { - const buffer = Buffer.from(data); - const bufferStream = new stream.PassThrough(); - bufferStream.end( buffer); - const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); - return bufferStream.pipe(zip).pipe(fs.createWriteStream(filename)); + return writeToFileIterable(filename, [data]); } - -export function writeToFileRx(filename: string, data: Observable): fs.WriteStream { - const sourceStream = rxToStream(data); - +export function writeToFileIterable(filename: string, data: Iterable): fs.WriteStream { + const sourceStream = iterableToStream(data); const writeStream = fs.createWriteStream(filename); const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough(); - return sourceStream.pipe(zip).pipe(writeStream); } -export function writeToFileRxP(filename: string, data: Observable): Promise { - const stream = writeToFileRx(filename, data); +export function writeToFileIterableP(filename: string, data: Iterable): Promise { + const stream = writeToFileIterable(filename, data); return new Promise((resolve, reject) => { stream.on('finish', () => resolve()); stream.on('error', (e: Error) => reject(e)); diff --git a/packages/cspell-lib/src/index.ts b/packages/cspell-lib/src/index.ts index 74b0e322f19..cf3050f9d74 100644 --- a/packages/cspell-lib/src/index.ts +++ b/packages/cspell-lib/src/index.ts @@ -1,2 +1,3 @@ export * from './file'; +export { toArray as asyncIterableToArray } from './async/asyncIterable'; diff --git a/packages/cspell-tools/package-lock.json b/packages/cspell-tools/package-lock.json index 8f9f994b190..658aaf52dcd 100644 --- a/packages/cspell-tools/package-lock.json +++ b/packages/cspell-tools/package-lock.json @@ -1,6 +1,6 @@ { "name": "cspell-tools", - "version": "3.0.8", + "version": "4.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -32,15 +32,6 @@ "rxjs-stream": "^3.0.1" } }, - "cspell-trie-lib": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/cspell-trie-lib/-/cspell-trie-lib-3.0.13.tgz", - "integrity": "sha512-v6fYv6GlGJhI7KhVlxjKeotcDqGA+FE0aae/TExwnwAcqMw9zHL9BcUQYPmrY1qxJZoWHhWIegXiphwW8LZC+g==", - "requires": { - "gensequence": "^2.1.2", - "js-xxhash": "^1.0.1" - } - }, "fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -114,11 +105,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, - "js-xxhash": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/js-xxhash/-/js-xxhash-1.0.1.tgz", - "integrity": "sha512-ociwKjHkCPjqNMmZtD7uUoL+AVhcSZX3WPmirRwXg+PzKD0v5x9K2yLpvkSglbThvbGN/TVA+3XFjPVolQwDpQ==" - }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", diff --git a/packages/cspell-tools/package.json b/packages/cspell-tools/package.json index 6a5976f4a03..3ff0379866f 100644 --- a/packages/cspell-tools/package.json +++ b/packages/cspell-tools/package.json @@ -1,6 +1,6 @@ { "name": "cspell-tools", - "version": "3.0.8", + "version": "4.0.0", "description": "Tools to assist with the development of cSpell", "typings": "dist/index.d.ts", "bin": { @@ -40,8 +40,8 @@ "homepage": "https://github.com/streetsidesoftware/cspell#readme", "dependencies": { "commander": "^2.20.0", - "cspell-lib": "^3.0.8", - "cspell-trie-lib": "^3.0.13", + "cspell-lib": "^4.0.0", + "cspell-trie-lib": "^4.0.0", "fs-extra": "^7.0.1", "gensequence": "^2.1.2", "glob": "^7.1.4", @@ -71,6 +71,6 @@ ] }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" } } diff --git a/packages/cspell-tools/src/compiler/fileReader.test.ts b/packages/cspell-tools/src/compiler/fileReader.test.ts deleted file mode 100644 index 3a22aab194e..00000000000 --- a/packages/cspell-tools/src/compiler/fileReader.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { expect } from 'chai'; -import * as fReader from './fileReader'; -import { of } from 'rxjs'; -import { toArray } from 'rxjs/operators'; -import * as fs from 'fs'; - -describe('Validate the fileReader', () => { - it('tests stringsToLines', () => { - const strings = of('a1\n2\n3\n4', '5\n6'); - return fReader.stringsToLinesRx(strings).pipe(toArray()).toPromise().then((a) => { - expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6']); - }); - }); - - it('tests stringsToLines trailing new line', () => { - const strings = of('a1\n2\n3\n4', '5\n6\n'); - return fReader.stringsToLinesRx(strings).pipe(toArray()).toPromise().then((a) => { - expect(a).to.be.deep.equal(['a1', '2', '3', '45', '6', '']); - }); - }); - - it('test the file reader', () => { - return fReader.stringsToLinesRx(fReader.textFileStreamRx(__filename)) - .pipe(toArray()) - .toPromise() - .then(lines => { - const actual = lines.join('\n').replace(/\r/g, ''); - const expected = fs.readFileSync(__filename, 'UTF-8').replace(/\r/g, ''); - expect(actual).to.equal(expected); - }); - }); - - it('test the lineReaderRx', () => { - return fReader.lineReaderRx(__filename) - .pipe(toArray()) - .toPromise() - .then(lines => { - const expected = fs.readFileSync(__filename, 'UTF-8').split(/\r?\n/); - expect(lines).to.deep.equal(expected); - }); - }); - - it('test missing file', () => { - return fReader.lineReaderRx(__filename + 'not.found') - .pipe(toArray()) - .toPromise() - .then( - () => { - expect('not to be here').to.be.true; - }, - (e) => { - expect(e).to.be.instanceof(Error); - expect(e.code).to.be.equal('ENOENT'); - } - ); - }); - -}); \ No newline at end of file diff --git a/packages/cspell-tools/src/compiler/fileReader.ts b/packages/cspell-tools/src/compiler/fileReader.ts deleted file mode 100644 index dd8de00159b..00000000000 --- a/packages/cspell-tools/src/compiler/fileReader.ts +++ /dev/null @@ -1 +0,0 @@ -export { lineReaderRx, textFileStreamRx, stringsToLinesRx } from 'cspell-lib'; diff --git a/packages/cspell-tools/src/compiler/fileWriter.test.ts b/packages/cspell-tools/src/compiler/fileWriter.test.ts index 0f12594f88a..a3d5de91dc8 100644 --- a/packages/cspell-tools/src/compiler/fileWriter.test.ts +++ b/packages/cspell-tools/src/compiler/fileWriter.test.ts @@ -1,29 +1,19 @@ import { expect } from 'chai'; import * as fileWriter from './fileWriter'; -import * as fileReader from './fileReader'; -import { from } from 'rxjs'; -import { flatMap, concatMap, reduce } from 'rxjs/operators'; import * as loremIpsum from 'lorem-ipsum'; import * as path from 'path'; -import { mkdirp } from 'fs-extra'; +import { mkdirp, readFile } from 'fs-extra'; describe('Validate the writer', () => { - it('tests writing an Rx.Observable and reading it back.', () => { + it('tests writing an Rx.Observable and reading it back.', async () => { // cspell:ignore éåáí const text = loremIpsum({ count: 1000, format: 'plain', units: 'words'}) + ' éåáí'; const data = text.split(/\b/); const filename = path.join(__dirname, '..', '..', 'temp', 'tests-writing-an-observable.txt'); - return from(mkdirp(path.dirname(filename))).pipe( - flatMap(() => { - return fileWriter.writeToFileRxP(filename, from(data)); - }), - concatMap(() => fileReader.textFileStreamRx(filename)), - reduce((a, b) => a + b), - ) - .toPromise() - .then(result => { - expect(result).to.equal(text); - }); + await mkdirp(path.dirname(filename)); + await fileWriter.writeToFileIterableP(filename, data); + const result = await readFile(filename, 'utf8'); + expect(result).to.equal(text); }); }); \ No newline at end of file diff --git a/packages/cspell-tools/src/compiler/fileWriter.ts b/packages/cspell-tools/src/compiler/fileWriter.ts index bfcd64f03b0..b9614a8f8f4 100644 --- a/packages/cspell-tools/src/compiler/fileWriter.ts +++ b/packages/cspell-tools/src/compiler/fileWriter.ts @@ -1,14 +1,17 @@ -import * as fs from 'fs-extra'; import { Sequence, genSequence } from 'gensequence'; import { iterableToStream } from 'hunspell-reader/dist/iterableToStream'; import { batch } from 'hunspell-reader/dist/util'; - -export { writeToFile, writeToFileRx, writeToFileRxP } from 'cspell-lib'; +import { writeToFileIterableP } from 'cspell-lib'; +export { writeToFile, writeToFileIterableP, writeToFileIterable } from 'cspell-lib'; export function writeSeqToFile(seq: Sequence, outFile: string | undefined): Promise { + if (outFile) { + return writeToFileIterableP(outFile, seq); + } + return new Promise((resolve, reject) => { let resolved = false; - const out = outFile ? fs.createWriteStream(outFile) : process.stdout; + const out = process.stdout; const bufferedSeq = genSequence(batch(seq, 500)).map(batch => batch.join('')); const dataStream = iterableToStream(bufferedSeq); const fileStream = dataStream.pipe(out); diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts index 3e42d91a67a..d5210610b6b 100644 --- a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts +++ b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts @@ -89,8 +89,9 @@ describe('Validate the wordListCompiler', function() { .then(words => { return Trie.importTrieRx(from(words)).pipe(take(1)).toPromise() .then(node => { - expect([...Trie.iteratorTrieWords(node)].sort()).to.be.deep - .equal(citiesResult.split('\n').filter(a => !!a).sort()); + const expected = citiesResult.split('\n').filter(a => !!a).sort(); + const words = [...Trie.iteratorTrieWords(node)].sort(); + expect(words).to.be.deep.equal(expected); }); }); }); @@ -101,16 +102,16 @@ function distinct(): (word: string) => boolean { return a => known.has(a) ? false : (known.add(a), true); } -const cities = `\ -New York -New Amsterdam -Los Angeles -San Francisco -New Delhi -Mexico City -London -Paris -`; +// const cities = `\ +// New York +// New Amsterdam +// Los Angeles +// San Francisco +// New Delhi +// Mexico City +// London +// Paris +// `; const citiesSorted = `\ London diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.ts b/packages/cspell-tools/src/compiler/wordListCompiler.ts index 5a27ffcd7d3..223bb08edc0 100644 --- a/packages/cspell-tools/src/compiler/wordListCompiler.ts +++ b/packages/cspell-tools/src/compiler/wordListCompiler.ts @@ -1,17 +1,14 @@ -import * as fs from 'fs'; import * as XRegExp from 'xregexp'; import { genSequence, Sequence } from 'gensequence'; import * as Text from './text'; -import { lineReaderRx } from './fileReader'; -import { writeToFileRxP} from 'cspell-lib'; -import { Observable, from } from 'rxjs'; -import { flatMap, map, bufferCount, distinct, toArray, filter } from 'rxjs/operators'; +import * as lib from 'cspell-lib'; import * as path from 'path'; import { mkdirp } from 'fs-extra'; import * as Trie from 'cspell-trie-lib'; import * as HR from 'hunspell-reader'; import { streamWordsFromFile } from './iterateWordsFromFile'; import { writeSeqToFile } from './fileWriter'; +import { uniqueFilter } from 'hunspell-reader/dist/util'; const regNonWordOrSpace = XRegExp("[^\\p{L}' ]+", 'gi'); const regExpSpaceOrDash = /(?:\s+)|(?:-+)/g; @@ -52,28 +49,24 @@ interface CompileWordListOptions { } export async function compileWordList(filename: string, destFilename: string, options: CompileWordListOptions): Promise { - const getWords = () => regHunspellFile.test(filename) ? readHunspellFiles(filename) : lineReaderRx(filename); + const getWords = () => regHunspellFile.test(filename) ? readHunspellFiles(filename) : lib.asyncIterableToArray(lib.lineReaderAsync(filename)); const destDir = path.dirname(destFilename); const pDir = mkdirp(destDir); - const stream = getWords().pipe( - options.splitWords ? compileWordListWithSplitRx : compileSimpleWordListRx, - filter(a => !!a), - distinct(), - options.sort ? sort : map(a => a) - ); + const compile = options.splitWords ? compileWordListWithSplitSeq : compileSimpleWordListSeq; - const buffered = stream.pipe( - map(a => a + '\n'), - bufferCount(1024), - map(a => a.join('')), - ); + const words = genSequence(await getWords()); + const seq = compile(words) + .filter(a => !!a) + .filter(uniqueFilter(10000)); + + const finalSeq = options.sort ? genSequence(sort(seq)) : seq; await pDir; - return writeToFileRxP(destFilename, buffered); + return writeSeqToFile(finalSeq.map(a => a + '\n'), destFilename); } @@ -82,27 +75,20 @@ export function compileWordListWithSplit(filename: string, destFilename: string) return compileWordList(filename, destFilename, { splitWords: true, sort: true }); } -export async function compileSimpleWordList(filename: string, destFilename: string, options: CompileWordListOptions): Promise { +export async function compileSimpleWordList(filename: string, destFilename: string, _options: CompileWordListOptions): Promise { return compileWordList(filename, destFilename, { splitWords: false, sort: true }); } -function sort(words: Observable): Observable { - return words.pipe( - toArray(), - flatMap(a => a.sort()), - ); +function sort(words: Iterable): Iterable { + return [...words].sort(); } -function compileWordListWithSplitRx(words: Observable): Observable { - return words.pipe( - flatMap(line => lineToWords(line).toArray()), - ); +function compileWordListWithSplitSeq(words: Sequence): Sequence { + return words.concatMap(line => lineToWords(line).toArray()); } -function compileSimpleWordListRx(words: Observable): Observable { - return words.pipe( - map(a => a.toLowerCase()), - ); +function compileSimpleWordListSeq(words: Sequence): Sequence { + return words.map(a => a.toLowerCase()); } export function normalizeWordsToTrie(words: Sequence): Trie.TrieNode { @@ -115,24 +101,20 @@ export async function compileWordListToTrieFile(words: Sequence, destFil const destDir = path.dirname(destFilename); const pDir = mkdirp(destDir); const pRoot = normalizeWordsToTrie(words); - const [_, root] = await Promise.all([pDir, pRoot]); + const [root] = await Promise.all([pRoot, pDir]); return writeSeqToFile(Trie.serializeTrie(root, { base: 32, comment: 'Built by cspell-tools.' }), destFilename); } const regHunspellFile = /\.(dic|aff)$/i; -function readHunspellFiles(filename: string): Observable { +async function readHunspellFiles(filename: string): Promise> { const dicFile = filename.replace(regHunspellFile, '.dic'); const affFile = filename.replace(regHunspellFile, '.aff'); - const reader = HR.HunspellReader.createFromFiles(affFile, dicFile); + const reader = await HR.IterableHunspellReader.createFromFiles(affFile, dicFile); - const r = from(reader).pipe( - flatMap(reader => reader.readWordsRx()), - map(aff => aff.word), - ); - return r; + return genSequence(reader.iterateWords()); } export async function compileTrie(filename: string, destFilename: string): Promise { diff --git a/packages/cspell-trie-lib/package-lock.json b/packages/cspell-trie-lib/package-lock.json index bd39938d2fd..303122c88eb 100644 --- a/packages/cspell-trie-lib/package-lock.json +++ b/packages/cspell-trie-lib/package-lock.json @@ -1,6 +1,6 @@ { "name": "cspell-trie-lib", - "version": "3.0.13", + "version": "4.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/packages/cspell-trie-lib/package.json b/packages/cspell-trie-lib/package.json index 7724e960b88..fb1dc4c1930 100644 --- a/packages/cspell-trie-lib/package.json +++ b/packages/cspell-trie-lib/package.json @@ -1,6 +1,6 @@ { "name": "cspell-trie-lib", - "version": "3.0.13", + "version": "4.0.0", "description": "Trie Data Structure to support cspell.", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -58,6 +58,6 @@ ] }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" } } diff --git a/packages/cspell-trie/.vscode/launch.json b/packages/cspell-trie/.vscode/launch.json index c69dbc30ce6..615f516c2ab 100644 --- a/packages/cspell-trie/.vscode/launch.json +++ b/packages/cspell-trie/.vscode/launch.json @@ -23,7 +23,7 @@ "type": "node", "request": "launch", "name": "Run Mocha Test Current File", - "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", + "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha", "args": [ "--timeout","999999", "--colors", diff --git a/packages/cspell-trie/package-lock.json b/packages/cspell-trie/package-lock.json index 7c35b272f73..977cc4b7201 100644 --- a/packages/cspell-trie/package-lock.json +++ b/packages/cspell-trie/package-lock.json @@ -1,6 +1,6 @@ { "name": "cspell-trie", - "version": "3.0.10", + "version": "4.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -9,15 +9,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==" }, - "cspell-trie-lib": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/cspell-trie-lib/-/cspell-trie-lib-3.0.13.tgz", - "integrity": "sha512-v6fYv6GlGJhI7KhVlxjKeotcDqGA+FE0aae/TExwnwAcqMw9zHL9BcUQYPmrY1qxJZoWHhWIegXiphwW8LZC+g==", - "requires": { - "gensequence": "^2.1.2", - "js-xxhash": "^1.0.1" - } - }, "fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -43,11 +34,6 @@ "resolved": "https://registry.npmjs.org/iterable-to-stream/-/iterable-to-stream-1.0.1.tgz", "integrity": "sha512-O62gD5ADMUGtJoOoM9U6LQ7i4byPXUNoHJ6mqsmkQJcom331ZJGDApWgDESWyBMEHEJRjtHozgIiTzYo9RU4UA==" }, - "js-xxhash": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/js-xxhash/-/js-xxhash-1.0.1.tgz", - "integrity": "sha512-ociwKjHkCPjqNMmZtD7uUoL+AVhcSZX3WPmirRwXg+PzKD0v5x9K2yLpvkSglbThvbGN/TVA+3XFjPVolQwDpQ==" - }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", diff --git a/packages/cspell-trie/package.json b/packages/cspell-trie/package.json index a466b730171..bc09641cf36 100644 --- a/packages/cspell-trie/package.json +++ b/packages/cspell-trie/package.json @@ -1,6 +1,6 @@ { "name": "cspell-trie", - "version": "3.0.10", + "version": "4.0.0", "description": "Trie Data Structure and tools to support cspell.", "bin": "./dist/app.js", "main": "dist/index.js", @@ -38,7 +38,7 @@ "homepage": "https://github.com/streetsidesoftware/cspell#readme", "dependencies": { "commander": "^2.20.0", - "cspell-trie-lib": "^3.0.13", + "cspell-trie-lib": "^4.0.0", "fs-extra": "^7.0.1", "gensequence": "^2.1.2", "iterable-to-stream": "^1.0.1" @@ -62,6 +62,6 @@ ] }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" } } diff --git a/packages/cspell-trie/src/index.test.ts b/packages/cspell-trie/src/index.test.ts new file mode 100644 index 00000000000..4574d9ba0ca --- /dev/null +++ b/packages/cspell-trie/src/index.test.ts @@ -0,0 +1,14 @@ +import {expect} from 'chai'; +import * as lib from './index'; +import { Trie } from 'cspell-trie-lib'; + +describe('Validate Library', () => { + it('test', () => { + const words = ['apple', 'banana', 'kiwi', 'mango', 'orange']; + const w = lib.createTriFromList(words); + const s = lib.serializeTrie(w); + const t = lib.importTrie(s); + const tt = new Trie(t); + expect([...tt.words()]).to.be.deep.equal(words); + }); +}); diff --git a/packages/cspell/CHANGELOG.md b/packages/cspell/CHANGELOG.md index 615109183bd..2d9892e4cbf 100644 --- a/packages/cspell/CHANGELOG.md +++ b/packages/cspell/CHANGELOG.md @@ -1,5 +1,8 @@ # Release Notes +## [4.0.0] +- **Breaking Change** drop support for Node 8 and 9. + ## [3.2.14] - Updated `package.json` references to point to the new monorepo - [Resolve paths beginning with tilde as $HOME by `tribut` · Pull Request #83](https://github.com/streetsidesoftware/cspell/pull/83) diff --git a/packages/cspell/package-lock.json b/packages/cspell/package-lock.json index 5b669d46759..b18a6146b58 100644 --- a/packages/cspell/package-lock.json +++ b/packages/cspell/package-lock.json @@ -1,6 +1,6 @@ { "name": "cspell", - "version": "3.2.17", + "version": "4.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -219,6 +219,52 @@ "configstore": "^4.0.0" } }, + "cspell-lib": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/cspell-lib/-/cspell-lib-3.0.8.tgz", + "integrity": "sha512-JRuvKQEli2SYYhH0/P4YgOEhZM9hKzKO8Fw1PPkX+nSiYWrdt7xkNd6vBG0t1aGngm7Tw4fBs1V3nl3D9BlvIA==", + "dev": true, + "requires": { + "iconv-lite": "^0.4.24", + "rxjs-stream": "^3.0.1" + } + }, + "cspell-tools": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/cspell-tools/-/cspell-tools-3.0.8.tgz", + "integrity": "sha512-524ZjYdwX2UTWmfZeFwX3DzAbX1Z5hy/Vi16L3EJD5OALgfhD2myobW5aArhXwG514Crs/JC/SD41NMFeWsmCA==", + "dev": true, + "requires": { + "commander": "^2.20.0", + "cspell-lib": "^3.0.8", + "cspell-trie": "^3.0.10", + "fs-extra": "^7.0.1", + "gensequence": "^2.1.2", + "glob": "^7.1.4", + "hunspell-reader": "^2.1.2", + "iconv-lite": "^0.4.24", + "minimatch": "^3.0.4", + "rxjs": "^6.5.2", + "rxjs-stream": "^3.0.1", + "xregexp": "^4.2.4" + } + }, + "cspell-trie": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/cspell-trie/-/cspell-trie-3.0.10.tgz", + "integrity": "sha512-70maLAmZTvbUo5qKr2UD3zA9FZ31p2a4bAyJsMdTvVHMj+t5CyuGGgOQ5579FF4MpOWpnSxT5/uzpZtWlkEk1A==", + "dev": true, + "requires": { + "commander": "^2.20.0", + "cspell-lib": "^3.0.8", + "fs-extra": "^7.0.1", + "gensequence": "^2.1.2", + "hunspell-reader": "^2.1.2", + "js-xxhash": "^1.0.1", + "rxjs": "^6.5.2", + "rxjs-stream": "^3.0.1" + } + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -285,6 +331,29 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, + "hunspell-reader": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/hunspell-reader/-/hunspell-reader-2.1.2.tgz", + "integrity": "sha512-RuaK4dpY1iUIazNxQcqSA5d5B0vcih0JPsb2X87nwxjqWSlM5fnckSVC9vLYrKddF/PL53fH9nQOqoHKvFndxg==", + "dev": true, + "requires": { + "commander": "^2.19.0", + "cspell-lib": "^3.0.5", + "fs-extra": "^7.0.1", + "gensequence": "^2.1.2", + "rxjs": "^6.3.3", + "rxjs-stream": "^3.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -309,6 +378,12 @@ "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, + "js-xxhash": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/js-xxhash/-/js-xxhash-1.0.1.tgz", + "integrity": "sha512-ociwKjHkCPjqNMmZtD7uUoL+AVhcSZX3WPmirRwXg+PzKD0v5x9K2yLpvkSglbThvbGN/TVA+3XFjPVolQwDpQ==", + "dev": true + }, "json-parser": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/json-parser/-/json-parser-1.1.5.tgz", @@ -372,6 +447,18 @@ "tslib": "^1.9.0" } }, + "rxjs-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rxjs-stream/-/rxjs-stream-3.0.2.tgz", + "integrity": "sha512-DASSTemCVcghTvUGKd6g2QYQ1Y/tCKwxZ2Xj41+PH0GHRLQLknwLIIHtPfbK1Cb7aorf0jZXAe123oSMHXLn7Q==", + "dev": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", diff --git a/packages/cspell/package.json b/packages/cspell/package.json index 8a7f18e45d0..52d9a5c51b4 100644 --- a/packages/cspell/package.json +++ b/packages/cspell/package.json @@ -1,6 +1,6 @@ { "name": "cspell", - "version": "3.2.17", + "version": "4.0.0", "description": "A Spelling Checker for Code!", "main": "dist/index.js", "typings": "dist/index.d.ts", @@ -82,8 +82,8 @@ "cspell-dict-python": "^1.0.9", "cspell-dict-rust": "^1.0.6", "cspell-dict-scala": "^1.0.6", - "cspell-lib": "^3.0.8", - "cspell-trie": "^3.0.10", + "cspell-trie-lib": "^4.0.0", + "cspell-lib": "^4.0.0", "fs-extra": "^7.0.1", "gensequence": "^2.1.2", "get-stdin": "^7.0.0", @@ -94,7 +94,7 @@ "xregexp": "^4.2.4" }, "engines": { - "node": ">=8.0.0" + "node": ">=10.0.0" }, "nyc": { "include": [ diff --git a/packages/cspell/src/SpellingDictionary/DictionaryLoader.ts b/packages/cspell/src/SpellingDictionary/DictionaryLoader.ts index 351e57db417..caa52b67f4c 100644 --- a/packages/cspell/src/SpellingDictionary/DictionaryLoader.ts +++ b/packages/cspell/src/SpellingDictionary/DictionaryLoader.ts @@ -1,11 +1,9 @@ -import { - loadWordsRx, - splitLineIntoWordsRx, -} from '../wordListHelper'; -import { SpellingDictionary, createSpellingDictionaryRx, createSpellingDictionaryTrie } from './SpellingDictionary'; +import { splitLineIntoWords, splitLineIntoCodeWords, loadWordsNoError } from '../wordListHelper'; +import { SpellingDictionary, createSpellingDictionaryTrie, createSpellingDictionary } from './SpellingDictionary'; import * as path from 'path'; -import {flatMap} from 'rxjs/operators'; import { ReplaceMap } from '../Settings'; +import { genSequence } from 'gensequence'; +import { readLines } from '../util/fileReader'; export interface LoadOptions { // Type of file: @@ -68,18 +66,27 @@ function load(uri: string, options: LoadOptions): Promise { } -function loadSimpleWordList(filename: string, options: LoadOptions) { - return createSpellingDictionaryRx(loadWordsRx(filename), path.basename(filename), filename, options); +async function loadSimpleWordList(filename: string, options: LoadOptions) { + try { + const lines = await readLines(filename); + return createSpellingDictionary(lines, path.basename(filename), filename, options); + } catch (e) { + return Promise.reject(e); + } } -function loadWordList(filename: string, options: LoadOptions) { - return createSpellingDictionaryRx(loadWordsRx(filename).pipe(flatMap(splitLineIntoWordsRx)), path.basename(filename), filename, options); +async function loadWordList(filename: string, options: LoadOptions) { + const lines = genSequence(await readLines(filename)); + const words = lines.concatMap(splitLineIntoWords); + return createSpellingDictionary(words, path.basename(filename), filename, options); } -function loadCodeWordList(filename: string, options: LoadOptions) { - return createSpellingDictionaryRx(loadWordsRx(filename).pipe(flatMap(splitLineIntoWordsRx)), path.basename(filename), filename, options); +async function loadCodeWordList(filename: string, options: LoadOptions) { + const lines = genSequence(await readLines(filename)); + const words = lines.concatMap(splitLineIntoCodeWords); + return createSpellingDictionary(words, path.basename(filename), filename, options); } -function loadTrie(filename: string, options: LoadOptions) { - return createSpellingDictionaryTrie(loadWordsRx(filename), path.basename(filename), filename, options); +async function loadTrie(filename: string, options: LoadOptions) { + return createSpellingDictionaryTrie(await loadWordsNoError(filename), path.basename(filename), filename, options); } diff --git a/packages/cspell/src/SpellingDictionary/SpellingDictionary.test.ts b/packages/cspell/src/SpellingDictionary/SpellingDictionary.test.ts index 7929e2cbca4..5b51ebb83d4 100644 --- a/packages/cspell/src/SpellingDictionary/SpellingDictionary.test.ts +++ b/packages/cspell/src/SpellingDictionary/SpellingDictionary.test.ts @@ -1,37 +1,16 @@ import { expect } from 'chai'; -import { createSpellingDictionaryRx, createSpellingDictionary, SpellingDictionaryFromSet, SpellingDictionaryFromTrie } from './SpellingDictionary'; -import { from } from 'rxjs'; +import { createSpellingDictionary, SpellingDictionaryFromSet, SpellingDictionaryFromTrie } from './SpellingDictionary'; import { Trie } from 'cspell-trie'; // cSpell:ignore aple describe('Verify building Dictionary', () => { - it('build from rx list', () => { + it('build from word list', async () => { const words = [ 'apple', 'ape', 'able', 'apple', 'banana', 'orange', 'pear', 'aim', 'approach', 'bear' ]; - return createSpellingDictionaryRx(from(words), 'test', 'test') - .then(dict => { - expect(dict).to.be.instanceof(SpellingDictionaryFromSet); - if (dict instanceof SpellingDictionaryFromSet) { - expect(dict.words).to.be.instanceof(Set); - expect(dict.trie.root.c).to.be.instanceof(Map); - } - expect(dict.has('apple')).to.be.true; - const suggestions = dict.suggest('aple').map(({word}) => word); - expect(suggestions).to.contain('apple'); - expect(suggestions).to.contain('ape'); - expect(suggestions).to.not.contain('banana'); - }); - }); - - it('build from word list', () => { - const words = [ - 'apple', 'ape', 'able', 'apple', 'banana', 'orange', 'pear', 'aim', 'approach', 'bear' - ]; - - const dict = createSpellingDictionary(words, 'words', 'test'); + const dict = await createSpellingDictionary(words, 'words', 'test'); expect(dict.name).to.be.equal('words'); expect(dict).to.be.instanceof(SpellingDictionaryFromSet); if (dict instanceof SpellingDictionaryFromSet) { @@ -45,12 +24,12 @@ describe('Verify building Dictionary', () => { expect(suggestions).to.not.contain('banana'); }); - it('Test compounds from word list', () => { + it('Test compounds from word list', async () => { const words = [ 'apple', 'ape', 'able', 'apple', 'banana', 'orange', 'pear', 'aim', 'approach', 'bear' ]; - const dict = createSpellingDictionary(words, 'words', 'test', { useCompounds: true }); + const dict = await createSpellingDictionary(words, 'words', 'test', { useCompounds: true }); expect(dict.has('apple')).to.be.true; // cspell:ignore applebanana applebananas applebananaorange expect(dict.has('applebanana')).to.be.true; @@ -72,32 +51,12 @@ describe('Verify building Dictionary', () => { expect(suggestions).to.not.contain('banana'); }); - it('build from rx list containing non-strings', () => { - const words = [ - 'apple', 'ape', 'able', , 'apple', 'banana', 'orange', 5, 'pear', 'aim', 'approach', 'bear' - ]; - - return createSpellingDictionaryRx(from(words as string[]), 'test', 'test') - .then(dict => { - expect(dict).to.be.instanceof(SpellingDictionaryFromSet); - if (dict instanceof SpellingDictionaryFromSet) { - expect(dict.words).to.be.instanceof(Set); - expect(dict.trie.root.c).to.be.instanceof(Map); - } - expect(dict.has('apple')).to.be.true; - const suggestions = dict.suggest('aple').map(({word}) => word); - expect(suggestions).to.contain('apple'); - expect(suggestions).to.contain('ape'); - expect(suggestions).to.not.contain('banana'); - }); - }); - - it('build from list containing non-strings', () => { + it('build from list containing non-strings', async () => { const words = [ 'apple', 'ape', 'able', , 'apple', 'banana', 'orange', 5, 'pear', 'aim', 'approach', 'bear' ]; - const dict = createSpellingDictionary(words as string[], 'words', 'test'); + const dict = await createSpellingDictionary(words as string[], 'words', 'test'); expect(dict.name).to.be.equal('words'); expect(dict).to.be.instanceof(SpellingDictionaryFromSet); if (dict instanceof SpellingDictionaryFromSet) { diff --git a/packages/cspell/src/SpellingDictionary/SpellingDictionary.ts b/packages/cspell/src/SpellingDictionary/SpellingDictionary.ts index 4b72dec39c4..83b34e1ee2c 100644 --- a/packages/cspell/src/SpellingDictionary/SpellingDictionary.ts +++ b/packages/cspell/src/SpellingDictionary/SpellingDictionary.ts @@ -1,8 +1,6 @@ import { genSequence } from 'gensequence'; -import { Observable } from 'rxjs'; -import { filter, map, reduce, take } from 'rxjs/operators'; import { IterableLike } from '../util/IterableLike'; -import { Trie, importTrieRx, SuggestionCollector, SuggestionResult, CompoundWordsMethod } from 'cspell-trie'; +import { Trie, importTrie, SuggestionCollector, SuggestionResult, CompoundWordsMethod } from 'cspell-trie-lib'; import { createMapper } from '../util/repMap'; import { ReplaceMap } from '../Settings'; @@ -86,12 +84,12 @@ export class SpellingDictionaryFromSet implements SpellingDictionary { } } -export function createSpellingDictionary( +export async function createSpellingDictionary( wordList: string[] | IterableLike, name: string, source: string, options?: SpellingDictionaryOptions -): SpellingDictionary { +): Promise { const words = new Set(genSequence(wordList) .filter(word => typeof word === 'string') .map(word => word.toLowerCase().trim()) @@ -100,22 +98,6 @@ export function createSpellingDictionary( return new SpellingDictionaryFromSet(words, name, options, source); } -export function createSpellingDictionaryRx( - words: Observable, - name: string, - source: string, - options?: SpellingDictionaryOptions -): Promise { - const promise = words.pipe( - filter(word => typeof word === 'string'), - map(word => word.toLowerCase().trim()), - filter(word => !!word), - reduce((words: Set, word: string) => words.add(word), new Set()), - map(words => new SpellingDictionaryFromSet(words, name, options, source)), - ).toPromise(); - return promise; -} - export class SpellingDictionaryFromTrie implements SpellingDictionary { static readonly unknownWordsLimit = 1000; private _size: number = 0; @@ -193,16 +175,13 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary { } } -export function createSpellingDictionaryTrie( - data: Observable, +export async function createSpellingDictionaryTrie( + data: IterableLike, name: string, source: string, options?: SpellingDictionaryOptions ): Promise { - const promise = importTrieRx(data).pipe( - map(node => new Trie(node)), - map(trie => new SpellingDictionaryFromTrie(trie, name, options, source)), - take(1), - ).toPromise(); - return promise; + const trieNode = importTrie(data); + const trie = new Trie(trieNode); + return new SpellingDictionaryFromTrie(trie, name, options, source); } diff --git a/packages/cspell/src/SpellingDictionary/SpellingDictionaryCollection.test.ts b/packages/cspell/src/SpellingDictionary/SpellingDictionaryCollection.test.ts index 2eaf7cdaa81..c569c7cda1e 100644 --- a/packages/cspell/src/SpellingDictionary/SpellingDictionaryCollection.test.ts +++ b/packages/cspell/src/SpellingDictionary/SpellingDictionaryCollection.test.ts @@ -1,19 +1,18 @@ import { expect } from 'chai'; import * as Trie from 'cspell-trie'; import { SpellingDictionaryCollection, createCollectionP, createCollection } from './SpellingDictionaryCollection'; -import { createSpellingDictionary, createSpellingDictionaryRx, SpellingDictionaryFromTrie, CompoundWordsMethod } from './SpellingDictionary'; -import { from } from 'rxjs'; +import { createSpellingDictionary, SpellingDictionaryFromTrie, CompoundWordsMethod } from './SpellingDictionary'; describe('Verify using multiple dictionaries', () => { const wordsA = ['', 'apple', 'banana', 'orange', 'pear', 'pineapple', 'mango', 'avocado', 'grape', 'strawberry', 'blueberry', 'blackberry']; const wordsB = ['ape', 'lion', 'tiger', 'elephant', 'monkey', 'gazelle', 'antelope', 'aardvark', 'hyena']; const wordsC = ['ant', 'snail', 'beetle', 'worm', 'stink bug', 'centipede', 'millipede', 'flea', 'fly']; - it('checks for existence', () => { - const dicts = [ + it('checks for existence', async () => { + const dicts = await Promise.all([ createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]; + ]); const dictCollection = new SpellingDictionaryCollection(dicts, 'test', ['Avocado']); expect(dictCollection.has('mango')).to.be.true; @@ -23,14 +22,14 @@ describe('Verify using multiple dictionaries', () => { expect(dictCollection.size).to.be.equal(wordsA.length - 1 + wordsB.length + wordsC.length); }); - it('checks for suggestions', () => { + it('checks for suggestions', async () => { const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); - const dicts = [ + const dicts = await Promise.all([ trie, createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]; + ]); const dictCollection = createCollection(dicts, 'test', ['Avocado']); const sugsForTango = dictCollection.suggest('tango', 10); @@ -40,16 +39,16 @@ describe('Verify using multiple dictionaries', () => { expect(sugsForTango.map(a => a.word).filter(a => a === 'mango')).to.be.deep.equal(['mango']); }); - it('checks for compound suggestions', () => { + it('checks for compound suggestions', async () => { // Add "wordsA" twice, once as a compound dictionary and once as a normal dictionary. const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); trie.options.useCompounds = true; - const dicts = [ + const dicts = await Promise.all([ trie, createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]; + ]); // cspell:ignore appletango applemango const dictCollection = createCollection(dicts, 'test', ['Avocado']); @@ -60,14 +59,14 @@ describe('Verify using multiple dictionaries', () => { expect(sugs).to.contain('apple mango'); }); - it('checks for compound suggestions', () => { + it('checks for compound suggestions', async () => { const trie = new SpellingDictionaryFromTrie(Trie.Trie.create(wordsA), 'wordsA'); - const dicts = [ + const dicts = await Promise.all([ trie, createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]; + ]); // cspell:ignore appletango applemango const dictCollection = createCollection(dicts, 'test', ['Avocado']); @@ -78,50 +77,49 @@ describe('Verify using multiple dictionaries', () => { expect(sugs).to.contain('apple mango'); }); - it('checks for suggestions with flagged words', () => { - const dicts = [ + it('checks for suggestions with flagged words', async () => { + const dicts = await Promise.all([ createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]; + ]); const dictCollection = createCollection(dicts, 'test', ['Avocado']); const sugs = dictCollection.suggest('avocado', 10); expect(sugs.map(r => r.word)).to.be.not.contain('avocado'); }); - it('checks for suggestions from mixed sources', () => { - return Promise.all([ - createSpellingDictionaryRx(from(wordsA), 'wordsA', 'test'), + it('checks for suggestions from mixed sources', async () => { + const dicts = await Promise.all([ + createSpellingDictionary(wordsA, 'wordsA', 'test'), createSpellingDictionary(wordsB, 'wordsB', 'test'), createSpellingDictionary(wordsC, 'wordsC', 'test'), - ]) - .then(dicts => { - const dictCollection = new SpellingDictionaryCollection(dicts, 'test', []); - expect(dictCollection.has('mango')); - expect(dictCollection.has('lion')); - expect(dictCollection.has('ant')); - - const sugsForTango = dictCollection.suggest('tango', 10); - expect(sugsForTango).to.be.not.empty; - expect(sugsForTango[0].word).to.be.equal('mango'); - // make sure there is only one mango suggestion. - expect(sugsForTango.map(a => a.word).filter(a => a === 'mango')).to.be.deep.equal(['mango']); - - // cspell:ignore cellipede - const sugsForCellipede = dictCollection.suggest('cellipede', 5); - expect(sugsForCellipede).to.not.be.empty; - expect(sugsForCellipede.map(s => s.word)).to.contain('centipede'); - expect(sugsForCellipede.map(s => s.word)).to.contain('millipede'); - }); + ]); + + const dictCollection = new SpellingDictionaryCollection(dicts, 'test', []); + expect(dictCollection.has('mango')); + expect(dictCollection.has('lion')); + expect(dictCollection.has('ant')); + + const sugsForTango = dictCollection.suggest('tango', 10); + expect(sugsForTango).to.be.not.empty; + expect(sugsForTango[0].word).to.be.equal('mango'); + // make sure there is only one mango suggestion. + expect(sugsForTango.map(a => a.word).filter(a => a === 'mango')).to.be.deep.equal(['mango']); + + // cspell:ignore cellipede + const sugsForCellipede = dictCollection.suggest('cellipede', 5); + expect(sugsForCellipede).to.not.be.empty; + expect(sugsForCellipede.map(s => s.word)).to.contain('centipede'); + expect(sugsForCellipede.map(s => s.word)).to.contain('millipede'); }); it('creates using createCollectionP', () => { const dicts = [ - Promise.resolve(createSpellingDictionary(wordsA, 'wordsA', 'test')), - Promise.resolve(createSpellingDictionary(wordsB, 'wordsB', 'test')), - Promise.resolve(createSpellingDictionary(wordsC, 'wordsC', 'test')), + createSpellingDictionary(wordsA, 'wordsA', 'test'), + createSpellingDictionary(wordsB, 'wordsB', 'test'), + createSpellingDictionary(wordsC, 'wordsC', 'test'), ]; return createCollectionP(dicts, 'test', []).then(dictCollection => { diff --git a/packages/cspell/src/index.ts b/packages/cspell/src/index.ts index 380c1d168c6..6b38d7442ad 100644 --- a/packages/cspell/src/index.ts +++ b/packages/cspell/src/index.ts @@ -19,7 +19,6 @@ export { export { CompoundWordsMethod, createSpellingDictionary, - createSpellingDictionaryRx, getDictionary, SpellingDictionary, SuggestionCollector, diff --git a/packages/cspell/src/textValidator.test.ts b/packages/cspell/src/textValidator.test.ts index ef919d51d2d..c4a688d1497 100644 --- a/packages/cspell/src/textValidator.test.ts +++ b/packages/cspell/src/textValidator.test.ts @@ -5,12 +5,12 @@ import { createCollection } from './SpellingDictionary'; import { createSpellingDictionary } from './SpellingDictionary'; import { FreqCounter } from './util/FreqCounter'; -// cSpell:enableCompoundWords +// cspell:ignore whiteberry redmango lightbrown redberry describe('Validate textValidator functions', () => { - it('tests hasWordCheck', () => { + it('tests hasWordCheck', async () => { // cspell:ignore redgreenblueyellow strawberrymangobanana redwhiteblue - const dictCol = getSpellingDictionaryCollection(); + const dictCol = await getSpellingDictionaryCollection(); expect(hasWordCheck(dictCol, 'brown', true)).to.be.true; expect(hasWordCheck(dictCol, 'white', true)).to.be.true; expect(hasWordCheck(dictCol, 'berry', true)).to.be.true; @@ -23,47 +23,47 @@ describe('Validate textValidator functions', () => { expect(hasWordCheck(dictCol, 'redwhiteblue', true)).to.be.true; }); - it('tests textValidator no word compounds', () => { - const dictCol = getSpellingDictionaryCollection(); + it('tests textValidator no word compounds', async () => { + const dictCol = await getSpellingDictionaryCollection(); const result = validateText(sampleText, dictCol, {}); const errors = result.map(wo => wo.text).toArray(); expect(errors).to.deep.equal(['giraffe', 'lightbrown', 'whiteberry', 'redberry']); }); - it('tests textValidator with word compounds', () => { - const dictCol = getSpellingDictionaryCollection(); + it('tests textValidator with word compounds', async () => { + const dictCol = await getSpellingDictionaryCollection(); const result = validateText(sampleText, dictCol, { allowCompoundWords: true }); const errors = result.map(wo => wo.text).toArray(); expect(errors).to.deep.equal(['giraffe', 'whiteberry']); }); // cSpell:ignore xxxkxxxx xxxbxxxx - it('tests ignoring words that consist of a single repeated letter', () => { - const dictCol = getSpellingDictionaryCollection(); + it('tests ignoring words that consist of a single repeated letter', async () => { + const dictCol = await getSpellingDictionaryCollection(); const text = ' tttt gggg xxxxxxx jjjjj xxxkxxxx xxxbxxxx \n' + sampleText; const result = validateText(text, dictCol, { allowCompoundWords: true }); const errors = result.map(wo => wo.text).toArray().sort(); expect(errors).to.deep.equal(['giraffe', 'whiteberry', 'xxxbxxxx', 'xxxkxxxx']); }); - it('tests trailing s, ed, ing, etc. are attached to the words', () => { - const dictEmpty = createSpellingDictionary([], 'empty', 'test'); + it('tests trailing s, ed, ing, etc. are attached to the words', async () => { + const dictEmpty = await createSpellingDictionary([], 'empty', 'test'); const text = 'We have PUBLISHed multiple FIXesToThePROBLEMs'; const result = validateText(text, dictEmpty, { allowCompoundWords: true }); const errors = result.map(wo => wo.text).toArray(); expect(errors).to.deep.equal(['have', 'Published', 'multiple', 'Fixes', 'Problems']); }); - it('tests trailing s, ed, ing, etc.', () => { - const dictWords = getSpellingDictionaryCollection(); + it('tests trailing s, ed, ing, etc.', async () => { + const dictWords = await getSpellingDictionaryCollection(); const text = 'We have PUBLISHed multiple FIXesToThePROBLEMs'; const result = validateText(text, dictWords, { allowCompoundWords: true }); const errors = result.map(wo => wo.text).toArray().sort(); expect(errors).to.deep.equal([]); }); - it('test contractions', () => { - const dictWords = getSpellingDictionaryCollection(); + it('test contractions', async () => { + const dictWords = await getSpellingDictionaryCollection(); // cspell:disable const text = `We should’ve done a better job, but we couldn\\'t have known.`; // cspell:enable @@ -72,14 +72,14 @@ describe('Validate textValidator functions', () => { expect(errors).to.deep.equal([]); }); - it('tests maxDuplicateProblems', () => { - const dict = createSpellingDictionary([], 'empty', 'test'); + it('tests maxDuplicateProblems', async () => { + const dict = await createSpellingDictionary([], 'empty', 'test'); const text = sampleText; const result = validateText(text, dict, { maxNumberOfProblems: 1000, maxDuplicateProblems: 1 }); const freq = FreqCounter.create(result.map(t => t.text)); expect(freq.total).to.be.equal(freq.counters.size); const words = freq.counters.keys(); - const dict2 = createSpellingDictionary(words, 'test', 'test'); + const dict2 = await createSpellingDictionary(words, 'test', 'test'); const result2 = [...validateText(text, dict2, { maxNumberOfProblems: 1000, maxDuplicateProblems: 1 })]; expect(result2.length).to.be.equal(0); }); @@ -104,14 +104,14 @@ describe('Validate textValidator functions', () => { }); -function getSpellingDictionaryCollection() { - const dicts = [ +async function getSpellingDictionaryCollection() { + const dicts = await Promise.all([ createSpellingDictionary(colors, 'colors', 'test'), createSpellingDictionary(fruit, 'fruit', 'test'), createSpellingDictionary(animals, 'animals', 'test'), createSpellingDictionary(insects, 'insects', 'test'), createSpellingDictionary(words, 'words', 'test', { repMap: [['’', "'"]]}), - ]; + ]); return createCollection(dicts, 'collection'); } diff --git a/packages/cspell/src/util/fileReader.test.ts b/packages/cspell/src/util/fileReader.test.ts index 807732c69fe..0532337684e 100644 --- a/packages/cspell/src/util/fileReader.test.ts +++ b/packages/cspell/src/util/fileReader.test.ts @@ -3,8 +3,7 @@ import * as fileReader from './fileReader'; describe('Validate file reader', () => { it('Catches errors for non-existent files', () => { - return fileReader.textFileStream('./non-existent.txt') - .toPromise() + return fileReader.readLines('./non-existent.txt') .then( () => { expect(true).to.be.false; diff --git a/packages/cspell/src/util/fileReader.ts b/packages/cspell/src/util/fileReader.ts index 94dbbde23f1..3891614587b 100644 --- a/packages/cspell/src/util/fileReader.ts +++ b/packages/cspell/src/util/fileReader.ts @@ -1,4 +1,12 @@ -export {lineReaderRx as lineReader} from 'cspell-lib'; -export {textFileStreamRx as textFileStream} from 'cspell-lib'; -export {stringsToLinesRx as stringsToLines} from 'cspell-lib'; +import { readFile } from 'cspell-lib'; +import { toIterableIterator } from './iterableIteratorLib'; +export async function readLines(filename: string, encoding: BufferEncoding = 'utf8') { + try { + + const content = await readFile(filename, encoding); + return toIterableIterator(content.split(/\r?\n/g)); + } catch (e) { + return Promise.reject(e); + } +} diff --git a/packages/cspell/src/util/iterableIteratorLib.test.ts b/packages/cspell/src/util/iterableIteratorLib.test.ts new file mode 100644 index 00000000000..be66f1cf7af --- /dev/null +++ b/packages/cspell/src/util/iterableIteratorLib.test.ts @@ -0,0 +1,19 @@ +import {expect} from 'chai'; +import { toIterableIterator, concatIterables } from './iterableIteratorLib'; + +describe('Validate Iterable Iterators', () => { + it('test toIterableIterator', () => { + const values = [1, 2, 3, 4]; + expect(toIterableIterator(values)).to.not.equal(values); + expect([...toIterableIterator(values)]).to.deep.equal(values); + }); + + it('test concatIterables', () => { + const values = [ + [1, 2, 3], + toIterableIterator([4, 5, 6]), + new Set([7, 8, 9]) + ]; + expect([...concatIterables(...values)]).to.be.deep.equal([1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); +}); diff --git a/packages/cspell/src/util/iterableIteratorLib.ts b/packages/cspell/src/util/iterableIteratorLib.ts new file mode 100644 index 00000000000..b3efbec79df --- /dev/null +++ b/packages/cspell/src/util/iterableIteratorLib.ts @@ -0,0 +1,11 @@ +import { IterableLike } from './IterableLike'; + +export function* toIterableIterator(i: IterableLike) { + yield* i; +} + +export function* concatIterables(...iterables: IterableLike[]) { + for (const i of iterables) { + yield *i; + } +} diff --git a/packages/cspell/src/wordListHelper.test.ts b/packages/cspell/src/wordListHelper.test.ts index be4cf604b1a..67359eed992 100644 --- a/packages/cspell/src/wordListHelper.test.ts +++ b/packages/cspell/src/wordListHelper.test.ts @@ -1,56 +1,39 @@ import {expect} from 'chai'; import * as wlh from './wordListHelper'; -import {toArray} from 'rxjs/operators'; describe('Validate wordListHelper', () => { - it('tests splitLineIntoWordsRx', () => { + it('tests splitLineIntoWords', () => { const line = 'New York City'; - return wlh.splitLineIntoWordsRx(line) - .pipe(toArray()) - .toPromise() - .then(words => { - expect(words).to.be.deep.equal([line, ...line.split(' ')]); - }); + const words = wlh.splitLineIntoWords(line); + expect([...words]).to.be.deep.equal([line, ...line.split(' ')]); }); - it('tests splitLineIntoCodeWordsRx', () => { + it('tests splitLineIntoCodeWords', () => { const line = 'cSpell:disableCompoundWords extra'; - return wlh.splitLineIntoCodeWordsRx(line) - .pipe(toArray()) - .toPromise() - .then(words => { - expect(words).to.be.deep.equal([ - 'cSpell', - 'disableCompoundWords', - 'extra', - 'c', - 'Spell', - 'disable', - 'Compound', - 'Words', - ]); - }); + const words = wlh.splitLineIntoCodeWords(line); + expect([...words]).to.be.deep.equal([ + 'cSpell', + 'disableCompoundWords', + 'extra', + 'c', + 'Spell', + 'disable', + 'Compound', + 'Words', + ]); }); it('tests splitLineIntoCodeWordsRx', () => { const line = 'New York City'; - return wlh.splitLineIntoCodeWordsRx(line) - .pipe(toArray()) - .toPromise() - .then(words => { - expect(words).to.be.deep.equal([ - 'New York City', - 'New', 'York', 'City', - ]); - }); + const words = wlh.splitLineIntoCodeWords(line); + expect([...words]).to.be.deep.equal([ + 'New York City', + 'New', 'York', 'City', + ]); }); - it('tests loadWordsRx error handling', () => { - return wlh.loadWordsRx('not_found.txt') - .pipe(toArray()) - .toPromise() - .then(values => { - expect(values).to.be.empty; - }); + it('tests loadWordsRx error handling', async () => { + const values = await wlh.loadWordsNoError('not_found.txt'); + expect([...values]).to.be.empty; }); }); diff --git a/packages/cspell/src/wordListHelper.ts b/packages/cspell/src/wordListHelper.ts index 6b9a6e85085..f6a60fe3cd6 100644 --- a/packages/cspell/src/wordListHelper.ts +++ b/packages/cspell/src/wordListHelper.ts @@ -1,9 +1,8 @@ // cSpell:enableCompoundWords -import { Observable, from } from 'rxjs'; -import { catchError } from 'rxjs/operators'; import * as Text from './util/text'; -import { lineReader } from './util/fileReader'; +import { readLines } from './util/fileReader'; import * as XRegExp from 'xregexp'; +import { toIterableIterator, concatIterables } from './util/iterableIteratorLib'; const regExpWordsWithSpaces = XRegExp('^\\s*\\p{L}+(?:\\s+\\p{L}+){0,3}$'); @@ -14,13 +13,14 @@ export interface WordDictionary { export type WordSet = Set; -export function loadWordsRx(filename: string): Observable { - return lineReader(filename) - .pipe(catchError((e: any) => { - logError(e); - return from([]); - })) - ; +/** + * Reads words from a file. It will not throw and error. + * @param filename the file to read + */ +export function loadWordsNoError(filename: string) { + return readLines(filename).catch( + e => (logError(e), toIterableIterator([])) + ); } function logError(e: any) { @@ -37,18 +37,15 @@ export function splitCodeWords(words: string[]) { .reduce((a, b) => a.concat(b), []); } -export function splitLineIntoCodeWordsRx(line: string): Observable { +export function splitLineIntoCodeWords(line: string): IterableIterator { const asMultiWord = regExpWordsWithSpaces.test(line) ? [ line ] : []; const asWords = splitLine(line); const splitWords = splitCodeWords(asWords); - const wordsToAdd = new Set([...asMultiWord, ...asWords, ...splitWords]); - return from([...wordsToAdd]); + const wordsToAdd = new Set(concatIterables(asMultiWord, asWords, splitWords)); + return toIterableIterator(wordsToAdd); } -export function splitLineIntoWordsRx(line: string): Observable { +export function splitLineIntoWords(line: string): IterableIterator { const asWords = splitLine(line); - const wordsToAdd = [line, ...asWords]; - return from(wordsToAdd); + return concatIterables([line], asWords); } - -