From f8ccc4aa4966457ebdfabaca2a8d33e221c53e42 Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Sat, 9 Apr 2022 07:28:39 +0200 Subject: [PATCH] fix: Ignore directories when checking files The following would cause an error when there was a subdirectory. ```sh ls -1 | cspell "**" --cache ``` --- .../cspell-pipe/src/operators/filter.test.ts | 56 ++++++++++++++++++- packages/cspell-pipe/src/operators/filter.ts | 14 ++++- packages/cspell/src/lint/lint.ts | 9 ++- packages/cspell/src/util/async.ts | 15 ++++- packages/cspell/src/util/fileHelper.test.ts | 37 +++++++++++- packages/cspell/src/util/fileHelper.ts | 49 ++++++++++------ 6 files changed, 151 insertions(+), 29 deletions(-) diff --git a/packages/cspell-pipe/src/operators/filter.test.ts b/packages/cspell-pipe/src/operators/filter.test.ts index da6b35e7cb4..a722228632d 100644 --- a/packages/cspell-pipe/src/operators/filter.test.ts +++ b/packages/cspell-pipe/src/operators/filter.test.ts @@ -1,6 +1,6 @@ -import { opFilter } from '.'; import { toArray, toAsyncIterable } from '../helpers'; import { pipeAsync, pipeSync } from '../pipe'; +import { opFilter, opFilterAsync } from './filter'; describe('Validate filter', () => { test('filter', async () => { @@ -13,7 +13,7 @@ describe('Validate filter', () => { const filterToLen = opFilter(filterFn); const s = pipeSync(values, filterToLen); - const a = pipeAsync(toAsyncIterable(values), filterToLen); + const a = pipeAsync(values, filterToLen); const sync = toArray(s); const async = await toArray(a); @@ -22,6 +22,58 @@ describe('Validate filter', () => { expect(async).toEqual(expected); }); + type Primitives = string | number | boolean; + type PromisePrim = Promise | Promise | Promise; + + test.each` + values | expected + ${[]} | ${[]} + ${[true, Promise.resolve(false), Promise.resolve(''), 'hello']} | ${[true, 'hello']} + ${[0, Promise.resolve('hey')]} | ${['hey']} + `( + 'filter async', + async ({ values, expected }: { values: (Primitives | PromisePrim)[]; expected: Primitives[] }) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const isTruthy = async (v: Primitives) => !!(await v); + const aValues = pipeAsync(values, opFilterAsync(isTruthy)); + const result = await toArray(aValues); + expect(result).toEqual(expected); + } + ); + + test('is a', async () => { + function isString(a: unknown): a is string { + return typeof a === 'string'; + } + const filtered = pipeAsync(['string', 4, {}, 'hello', undefined], opFilterAsync(isString)); + + expect(await toArray(filtered)).toEqual(['string', 'hello']); + }); + + test('async filter', async () => { + function isString(a: unknown): a is string { + return typeof a === 'string'; + } + async function truthyAsync(a: unknown): Promise { + return !!(await a); + } + const filtered = pipeAsync( + [ + Promise.resolve('string'), + '', + 4, + {}, + 'hello', + Promise.resolve(undefined), + Promise.resolve(Promise.resolve(Promise.resolve('deep'))), + ], + opFilterAsync(truthyAsync), + opFilter(isString) + ); + + expect(await toArray(filtered)).toEqual(['string', 'hello', 'deep']); + }); + test('filter isDefined', async () => { const values = ['a', 'b', undefined, 'c', 'd']; const expected = values.filter(isDefined); diff --git a/packages/cspell-pipe/src/operators/filter.ts b/packages/cspell-pipe/src/operators/filter.ts index ced0e530880..1ff29e65034 100644 --- a/packages/cspell-pipe/src/operators/filter.ts +++ b/packages/cspell-pipe/src/operators/filter.ts @@ -1,12 +1,20 @@ import { isAsyncIterable } from '../helpers/util'; import { PipeFn } from '../internalTypes'; +// prettier-ignore export function opFilterAsync(filterFn: (v: T) => v is S): (iter: AsyncIterable) => AsyncIterable; -export function opFilterAsync(filterFn: (v: T) => boolean): (iter: AsyncIterable) => AsyncIterable; -export function opFilterAsync(filterFn: (v: T) => boolean): (iter: AsyncIterable) => AsyncIterable { +// prettier-ignore +export function opFilterAsync>(filterFn: (v: Awaited) => v is S): (iter: AsyncIterable) => AsyncIterable; +// prettier-ignore +export function opFilterAsync(filterFn: (v: Awaited) => boolean): (iter: AsyncIterable) => AsyncIterable>; +// prettier-ignore +export function opFilterAsync(filterFn: (v: Awaited) => Promise): (iter: AsyncIterable) => AsyncIterable>; +// prettier-ignore +export function opFilterAsync(filterFn: (v: Awaited) => boolean | Promise): (iter: AsyncIterable) => AsyncIterable> { async function* fn(iter: Iterable | AsyncIterable) { for await (const v of iter) { - if (filterFn(v)) yield v; + const pass = await filterFn(v); + if (pass) yield v; } } diff --git a/packages/cspell/src/lint/lint.ts b/packages/cspell/src/lint/lint.ts index 10f92e58603..4272ea0f039 100644 --- a/packages/cspell/src/lint/lint.ts +++ b/packages/cspell/src/lint/lint.ts @@ -1,4 +1,4 @@ -import { isAsyncIterable, opFilter, pipeAsync, pipeSync } from '@cspell/cspell-pipe'; +import { isAsyncIterable, operators, opFilter, pipeAsync, pipeSync } from '@cspell/cspell-pipe'; import type { CSpellReporter, CSpellSettings, Glob, Issue, RunResult, TextDocumentOffset } from '@cspell/cspell-types'; import { MessageTypes } from '@cspell/cspell-types'; import { findRepoRoot, GitIgnore } from 'cspell-gitignore'; @@ -16,6 +16,7 @@ import { fileInfoToDocument, FileResult, findFiles, + isNotDir, readConfig, readFileInfo, readFileListFiles, @@ -31,6 +32,8 @@ import chalk = require('chalk'); const npmPackage = require('../../package.json'); const version = npmPackage.version; +const { opFilterAsync } = operators; + export async function runLint(cfg: LintRequest): Promise { let { reporter } = cfg; cspell.setLogger(getLoggerFromReporter(reporter)); @@ -515,7 +518,7 @@ async function useFileLists( } const globMatcher = new GlobMatcher(includeGlobPatterns, options); - const files = await readFileListFiles(fileListFiles); const filterFiles = (file: string) => globMatcher.match(file); - return files instanceof Array ? files.filter(filterFiles) : pipeAsync(files, opFilter(filterFiles)); + const files = readFileListFiles(fileListFiles); + return pipeAsync(files, opFilter(filterFiles), opFilterAsync(isNotDir)); } diff --git a/packages/cspell/src/util/async.ts b/packages/cspell/src/util/async.ts index dd648919818..347e0ace46f 100644 --- a/packages/cspell/src/util/async.ts +++ b/packages/cspell/src/util/async.ts @@ -1,5 +1,16 @@ +import { operators } from '@cspell/cspell-pipe'; + export { - toAsyncIterable as mergeAsyncIterables, + helpers as asyncHelpers, + operators as asyncOperators, + pipeAsync as asyncPipe, toArray as asyncIterableToArray, - opMap as asyncMap, + toAsyncIterable as mergeAsyncIterables, } from '@cspell/cspell-pipe'; + +export const { + opMapAsync: asyncMap, + opFilterAsync: asyncFilter, + opAwaitAsync: asyncAwait, + opFlattenAsync: asyncFlatten, +} = operators; diff --git a/packages/cspell/src/util/fileHelper.test.ts b/packages/cspell/src/util/fileHelper.test.ts index b3169668cb9..d2f679bb302 100644 --- a/packages/cspell/src/util/fileHelper.test.ts +++ b/packages/cspell/src/util/fileHelper.test.ts @@ -1,6 +1,7 @@ -import { readFileInfo, readFileListFile, readFileListFiles } from './fileHelper'; +import { isDir, isFile, isNotDir, readFileInfo, readFileListFile, readFileListFiles } from './fileHelper'; import * as path from 'path'; import { IOError } from './errors'; +import { asyncIterableToArray } from './async'; const packageRoot = path.join(__dirname, '../..'); const fixtures = path.join(packageRoot, 'fixtures/fileHelper'); @@ -28,12 +29,12 @@ describe('fileHelper', () => { test('readFileListFiles', async () => { const files = ['file1', '../file2', 'dir/file3', 'nested/file2.txt']; - const r = await readFileListFiles([fileListFile, fileListFile2]); + const r = await asyncIterableToArray(readFileListFiles([fileListFile, fileListFile2])); expect(r).toEqual(files.map((f) => path.resolve(fixtures, f))); }); test('readFileListFiles Error', () => { - const r = readFileListFiles(['not-found.txt']); + const r = asyncIterableToArray(readFileListFiles(['not-found.txt'])); return expect(r).rejects.toEqual(oc({ message: 'Error reading file list from: "not-found.txt"' })); }); @@ -56,4 +57,34 @@ describe('fileHelper', () => { filename = r(__dirname, filename); await expect(readFileInfo(filename, undefined, false)).rejects.toThrow(expected); }); + + test.each` + filename | expected + ${__filename} | ${true} + ${__dirname} | ${false} + ${'not_found'} | ${false} + `('isFile $filename', async ({ filename, expected }) => { + filename = r(__dirname, filename); + expect(await isFile(filename)).toBe(expected); + }); + + test.each` + filename | expected + ${__filename} | ${false} + ${__dirname} | ${true} + ${'not_found'} | ${false} + `('isDir $filename', async ({ filename, expected }) => { + filename = r(__dirname, filename); + expect(await isDir(filename)).toBe(expected); + }); + + test.each` + filename | expected + ${__filename} | ${true} + ${__dirname} | ${false} + ${'not_found'} | ${true} + `('isDir $filename', async ({ filename, expected }) => { + filename = r(__dirname, filename); + expect(await isNotDir(filename)).toBe(expected); + }); }); diff --git a/packages/cspell/src/util/fileHelper.ts b/packages/cspell/src/util/fileHelper.ts index 0bd41131a5f..956bf2c9b0a 100644 --- a/packages/cspell/src/util/fileHelper.ts +++ b/packages/cspell/src/util/fileHelper.ts @@ -1,16 +1,15 @@ import * as cspell from 'cspell-lib'; -import * as fsp from 'fs-extra'; +import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib'; +import { promises as fsp } from 'fs'; import getStdin from 'get-stdin'; -import { GlobOptions, globP } from './glob'; import * as path from 'path'; -import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib'; +import { asyncAwait, asyncFlatten, asyncMap, asyncPipe, mergeAsyncIterables } from './async'; import { IOError, toApplicationError, toError } from './errors'; -import { mergeAsyncIterables, asyncMap } from './async'; +import { GlobOptions, globP } from './glob'; import { readStdin } from './stdin'; const UTF8: BufferEncoding = 'utf8'; const STDIN = 'stdin'; - export interface ConfigInfo { source: string; config: CSpellUserSettings; @@ -73,7 +72,7 @@ interface ReadFileInfoResult extends FileInfo { export function readFileInfo( filename: string, - encoding: string = UTF8, + encoding: BufferEncoding = UTF8, handleNotFound = false ): Promise { const pText = filename === STDIN ? getStdin() : fsp.readFile(filename, encoding); @@ -90,7 +89,7 @@ export function readFileInfo( ); } -export function readFile(filename: string, encoding: string = UTF8): Promise { +export function readFile(filename: string, encoding: BufferEncoding = UTF8): Promise { return readFileInfo(filename, encoding).then((info) => info.text); } @@ -147,16 +146,22 @@ const resolveFilenames = asyncMap(resolveFilename); * file will be resolved relative to the containing file. * @returns - a list of files to be processed. */ -export async function readFileListFiles(listFiles: string[]): Promise> { +export function readFileListFiles(listFiles: string[]): AsyncIterable { let useStdin = false; const files = listFiles.filter((file) => { const isStdin = file === 'stdin'; useStdin = useStdin || isStdin; return !isStdin; }); - const found = flatten(await Promise.all(files.map(readFileListFile))); + const found = asyncPipe( + files, + asyncMap((file) => readFileListFile(file)), + asyncAwait(), + asyncFlatten() + ); // Move `stdin` to the end. - return useStdin ? resolveFilenames(mergeAsyncIterables(found, readStdin())) : found; + const stdin = useStdin ? readStdin() : []; + return asyncPipe(mergeAsyncIterables(found, stdin), resolveFilenames); } /** @@ -180,12 +185,24 @@ export async function readFileListFile(listFile: string): Promise { } } -function flatten(fileLists: string[][]): string[] { - function* f() { - for (const list of fileLists) { - yield* list; - } +export async function isFile(filename: string): Promise { + try { + const stat = await fsp.stat(filename); + return stat.isFile(); + } catch (e) { + return false; + } +} + +export async function isDir(filename: string): Promise { + try { + const stat = await fsp.stat(filename); + return stat.isDirectory(); + } catch (e) { + return false; } +} - return [...f()]; +export function isNotDir(filename: string): Promise { + return isDir(filename).then((a) => !a); }