Skip to content

Commit

Permalink
fix: Ignore directories when checking files
Browse files Browse the repository at this point in the history 
The following would cause an error when there was a subdirectory.

```sh
ls -1 | cspell "**" --cache
```
  • Loading branch information
@Jason3S
Jason3S committed Apr 9, 2022
1 parent d23dc0a commit f8ccc4a
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 29 deletions.
56 changes: 54 additions & 2 deletions packages/cspell-pipe/src/operators/filter.test.ts
View file
@@ -1,6 +1,6 @@
import { opFilter } from '.';
import { toArray, toAsyncIterable } from '../helpers';
import { pipeAsync, pipeSync } from '../pipe';
import { opFilter, opFilterAsync } from './filter';

describe('Validate filter', () => {
test('filter', async () => {
Expand All @@ -13,7 +13,7 @@ describe('Validate filter', () => {
const filterToLen = opFilter(filterFn);

const s = pipeSync(values, filterToLen);
const a = pipeAsync(toAsyncIterable(values), filterToLen);
const a = pipeAsync(values, filterToLen);

const sync = toArray(s);
const async = await toArray(a);
Expand All @@ -22,6 +22,58 @@ describe('Validate filter', () => {
expect(async).toEqual(expected);
});

type Primitives = string | number | boolean;
type PromisePrim = Promise<string> | Promise<number> | Promise<boolean>;

test.each`
values | expected
${[]} | ${[]}
${[true, Promise.resolve(false), Promise.resolve(''), 'hello']} | ${[true, 'hello']}
${[0, Promise.resolve('hey')]} | ${['hey']}
`(
'filter async',
async ({ values, expected }: { values: (Primitives | PromisePrim)[]; expected: Primitives[] }) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const isTruthy = async (v: Primitives) => !!(await v);
const aValues = pipeAsync(values, opFilterAsync<Primitives | PromisePrim>(isTruthy));
const result = await toArray(aValues);
expect(result).toEqual(expected);
}
);

test('is a', async () => {
function isString(a: unknown): a is string {
return typeof a === 'string';
}
const filtered = pipeAsync(['string', 4, {}, 'hello', undefined], opFilterAsync(isString));

expect(await toArray(filtered)).toEqual(['string', 'hello']);
});

test('async filter', async () => {
function isString(a: unknown): a is string {
return typeof a === 'string';
}
async function truthyAsync(a: unknown): Promise<boolean> {
return !!(await a);
}
const filtered = pipeAsync(
[
Promise.resolve('string'),
'',
4,
{},
'hello',
Promise.resolve(undefined),
Promise.resolve(Promise.resolve(Promise.resolve('deep'))),
],
opFilterAsync(truthyAsync),
opFilter(isString)
);

expect(await toArray(filtered)).toEqual(['string', 'hello', 'deep']);
});

test('filter isDefined', async () => {
const values = ['a', 'b', undefined, 'c', 'd'];
const expected = values.filter(isDefined);
Expand Down
14 changes: 11 additions & 3 deletions packages/cspell-pipe/src/operators/filter.ts
View file
@@ -1,12 +1,20 @@
import { isAsyncIterable } from '../helpers/util';
import { PipeFn } from '../internalTypes';

// prettier-ignore
export function opFilterAsync<T, S extends T>(filterFn: (v: T) => v is S): (iter: AsyncIterable<T>) => AsyncIterable<S>;
export function opFilterAsync<T>(filterFn: (v: T) => boolean): (iter: AsyncIterable<T>) => AsyncIterable<T>;
export function opFilterAsync<T>(filterFn: (v: T) => boolean): (iter: AsyncIterable<T>) => AsyncIterable<T> {
// prettier-ignore
export function opFilterAsync<T, S extends Awaited<T>>(filterFn: (v: Awaited<T>) => v is S): (iter: AsyncIterable<T>) => AsyncIterable<S>;
// prettier-ignore
export function opFilterAsync<T>(filterFn: (v: Awaited<T>) => boolean): (iter: AsyncIterable<T>) => AsyncIterable<Awaited<T>>;
// prettier-ignore
export function opFilterAsync<T>(filterFn: (v: Awaited<T>) => Promise<boolean>): (iter: AsyncIterable<T>) => AsyncIterable<Awaited<T>>;
// prettier-ignore
export function opFilterAsync<T>(filterFn: (v: Awaited<T>) => boolean | Promise<boolean>): (iter: AsyncIterable<T>) => AsyncIterable<Awaited<T>> {
async function* fn(iter: Iterable<T> | AsyncIterable<T>) {
for await (const v of iter) {
if (filterFn(v)) yield v;
const pass = await filterFn(v);
if (pass) yield v;
}
}

Expand Down
9 changes: 6 additions & 3 deletions packages/cspell/src/lint/lint.ts
View file
@@ -1,4 +1,4 @@
import { isAsyncIterable, opFilter, pipeAsync, pipeSync } from '@cspell/cspell-pipe';
import { isAsyncIterable, operators, opFilter, pipeAsync, pipeSync } from '@cspell/cspell-pipe';
import type { CSpellReporter, CSpellSettings, Glob, Issue, RunResult, TextDocumentOffset } from '@cspell/cspell-types';
import { MessageTypes } from '@cspell/cspell-types';
import { findRepoRoot, GitIgnore } from 'cspell-gitignore';
Expand All @@ -16,6 +16,7 @@ import {
fileInfoToDocument,
FileResult,
findFiles,
isNotDir,
readConfig,
readFileInfo,
readFileListFiles,
Expand All @@ -31,6 +32,8 @@ import chalk = require('chalk');
const npmPackage = require('../../package.json');
const version = npmPackage.version;

const { opFilterAsync } = operators;

export async function runLint(cfg: LintRequest): Promise<RunResult> {
let { reporter } = cfg;
cspell.setLogger(getLoggerFromReporter(reporter));
Expand Down Expand Up @@ -515,7 +518,7 @@ async function useFileLists(
}
const globMatcher = new GlobMatcher(includeGlobPatterns, options);

const files = await readFileListFiles(fileListFiles);
const filterFiles = (file: string) => globMatcher.match(file);
return files instanceof Array ? files.filter(filterFiles) : pipeAsync(files, opFilter(filterFiles));
const files = readFileListFiles(fileListFiles);
return pipeAsync(files, opFilter(filterFiles), opFilterAsync(isNotDir));
}
15 changes: 13 additions & 2 deletions packages/cspell/src/util/async.ts
View file
@@ -1,5 +1,16 @@
import { operators } from '@cspell/cspell-pipe';

export {
toAsyncIterable as mergeAsyncIterables,
helpers as asyncHelpers,
operators as asyncOperators,
pipeAsync as asyncPipe,
toArray as asyncIterableToArray,
opMap as asyncMap,
toAsyncIterable as mergeAsyncIterables,
} from '@cspell/cspell-pipe';

export const {
opMapAsync: asyncMap,
opFilterAsync: asyncFilter,
opAwaitAsync: asyncAwait,
opFlattenAsync: asyncFlatten,
} = operators;
37 changes: 34 additions & 3 deletions packages/cspell/src/util/fileHelper.test.ts
View file
@@ -1,6 +1,7 @@
import { readFileInfo, readFileListFile, readFileListFiles } from './fileHelper';
import { isDir, isFile, isNotDir, readFileInfo, readFileListFile, readFileListFiles } from './fileHelper';
import * as path from 'path';
import { IOError } from './errors';
import { asyncIterableToArray } from './async';

const packageRoot = path.join(__dirname, '../..');
const fixtures = path.join(packageRoot, 'fixtures/fileHelper');
Expand Down Expand Up @@ -28,12 +29,12 @@ describe('fileHelper', () => {

test('readFileListFiles', async () => {
const files = ['file1', '../file2', 'dir/file3', 'nested/file2.txt'];
const r = await readFileListFiles([fileListFile, fileListFile2]);
const r = await asyncIterableToArray(readFileListFiles([fileListFile, fileListFile2]));
expect(r).toEqual(files.map((f) => path.resolve(fixtures, f)));
});

test('readFileListFiles Error', () => {
const r = readFileListFiles(['not-found.txt']);
const r = asyncIterableToArray(readFileListFiles(['not-found.txt']));
return expect(r).rejects.toEqual(oc({ message: 'Error reading file list from: "not-found.txt"' }));
});

Expand All @@ -56,4 +57,34 @@ describe('fileHelper', () => {
filename = r(__dirname, filename);
await expect(readFileInfo(filename, undefined, false)).rejects.toThrow(expected);
});

test.each`
filename | expected
${__filename} | ${true}
${__dirname} | ${false}
${'not_found'} | ${false}
`('isFile $filename', async ({ filename, expected }) => {
filename = r(__dirname, filename);
expect(await isFile(filename)).toBe(expected);
});

test.each`
filename | expected
${__filename} | ${false}
${__dirname} | ${true}
${'not_found'} | ${false}
`('isDir $filename', async ({ filename, expected }) => {
filename = r(__dirname, filename);
expect(await isDir(filename)).toBe(expected);
});

test.each`
filename | expected
${__filename} | ${true}
${__dirname} | ${false}
${'not_found'} | ${true}
`('isDir $filename', async ({ filename, expected }) => {
filename = r(__dirname, filename);
expect(await isNotDir(filename)).toBe(expected);
});
});
49 changes: 33 additions & 16 deletions packages/cspell/src/util/fileHelper.ts
View file
@@ -1,16 +1,15 @@
import * as cspell from 'cspell-lib';
import * as fsp from 'fs-extra';
import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib';
import { promises as fsp } from 'fs';
import getStdin from 'get-stdin';
import { GlobOptions, globP } from './glob';
import * as path from 'path';
import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib';
import { asyncAwait, asyncFlatten, asyncMap, asyncPipe, mergeAsyncIterables } from './async';
import { IOError, toApplicationError, toError } from './errors';
import { mergeAsyncIterables, asyncMap } from './async';
import { GlobOptions, globP } from './glob';
import { readStdin } from './stdin';

const UTF8: BufferEncoding = 'utf8';
const STDIN = 'stdin';

export interface ConfigInfo {
source: string;
config: CSpellUserSettings;
Expand Down Expand Up @@ -73,7 +72,7 @@ interface ReadFileInfoResult extends FileInfo {

export function readFileInfo(
filename: string,
encoding: string = UTF8,
encoding: BufferEncoding = UTF8,
handleNotFound = false
): Promise<ReadFileInfoResult> {
const pText = filename === STDIN ? getStdin() : fsp.readFile(filename, encoding);
Expand All @@ -90,7 +89,7 @@ export function readFileInfo(
);
}

export function readFile(filename: string, encoding: string = UTF8): Promise<string> {
export function readFile(filename: string, encoding: BufferEncoding = UTF8): Promise<string> {
return readFileInfo(filename, encoding).then((info) => info.text);
}

Expand Down Expand Up @@ -147,16 +146,22 @@ const resolveFilenames = asyncMap(resolveFilename);
* file will be resolved relative to the containing file.
* @returns - a list of files to be processed.
*/
export async function readFileListFiles(listFiles: string[]): Promise<string[] | AsyncIterable<string>> {
export function readFileListFiles(listFiles: string[]): AsyncIterable<string> {
let useStdin = false;
const files = listFiles.filter((file) => {
const isStdin = file === 'stdin';
useStdin = useStdin || isStdin;
return !isStdin;
});
const found = flatten(await Promise.all(files.map(readFileListFile)));
const found = asyncPipe(
files,
asyncMap((file) => readFileListFile(file)),
asyncAwait(),
asyncFlatten()
);
// Move `stdin` to the end.
return useStdin ? resolveFilenames(mergeAsyncIterables(found, readStdin())) : found;
const stdin = useStdin ? readStdin() : [];
return asyncPipe(mergeAsyncIterables(found, stdin), resolveFilenames);
}

/**
Expand All @@ -180,12 +185,24 @@ export async function readFileListFile(listFile: string): Promise<string[]> {
}
}

function flatten(fileLists: string[][]): string[] {
function* f() {
for (const list of fileLists) {
yield* list;
}
export async function isFile(filename: string): Promise<boolean> {
try {
const stat = await fsp.stat(filename);
return stat.isFile();
} catch (e) {
return false;
}
}

export async function isDir(filename: string): Promise<boolean> {
try {
const stat = await fsp.stat(filename);
return stat.isDirectory();
} catch (e) {
return false;
}
}

return [...f()];
export function isNotDir(filename: string): Promise<boolean> {
return isDir(filename).then((a) => !a);
}

0 comments on commit f8ccc4a

Please sign in to comment.