Skip to content

Commit

Permalink
Merge 1d19653 into 5554ecb
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jason3S
Jason3S committed Jan 26, 2022
2 parents 5554ecb + 1d19653 commit 8a03b8a
Show file tree
Hide file tree
Showing 16 changed files with 406 additions and 104 deletions.
33 changes: 26 additions & 7 deletions packages/cspell-gitignore/src/GitIgnore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,23 @@ export class GitIgnore {
this.resolvedGitIgnoreHierarchies.set(directory, found);
return find;
}
filterOutIgnored(files: string[]): Promise<string[]>;
filterOutIgnored(files: Iterable<string>): Promise<string[]>;
filterOutIgnored(files: AsyncIterable<string>): AsyncIterable<string>;
filterOutIgnored(files: Iterable<string> | AsyncIterable<string>): Promise<string[]> | AsyncIterable<string>;
filterOutIgnored(files: Iterable<string> & AsyncIterable<string>): AsyncIterable<string>;
filterOutIgnored(files: Iterable<string> | AsyncIterable<string>): Promise<string[]> | AsyncIterable<string> {
const iter = this.filterOutIgnoredAsync(files);
return isAsyncIterable(files) ? iter : asyncIterableToArray(iter);
}

async filterOutIgnored(files: string[]): Promise<string[]> {
const result: string[] = [];

for (const file of files) {
async *filterOutIgnoredAsync(files: Iterable<string> | AsyncIterable<string>): AsyncIterable<string> {
for await (const file of files) {
const isIgnored = this.isIgnoredQuick(file) ?? (await this.isIgnored(file));
if (!isIgnored) {
result.push(file);
yield file;
}
}

return result;
}

get roots(): string[] {
Expand Down Expand Up @@ -125,3 +130,17 @@ function sortRoots(roots: string[]): string[] {
roots.sort((a, b) => a.length - b.length);
return roots;
}

function isAsyncIterable<T>(i: Iterable<T> | AsyncIterable<T>): i is AsyncIterable<T> {
const as = <AsyncIterable<T>>i;
return typeof as[Symbol.asyncIterator] === 'function';
}

async function asyncIterableToArray<T>(iter: Iterable<T> | AsyncIterable<T>): Promise<Awaited<T>[]> {
const r: Awaited<T>[] = [];

for await (const t of iter) {
r.push(t);
}
return r;
}
2 changes: 1 addition & 1 deletion packages/cspell/src/app.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import * as Util from 'util';
import { URI } from 'vscode-uri';
import * as app from './app';
import * as Link from './link';
import { mergeAsyncIterables } from './util/util';
import { mergeAsyncIterables } from './util/async';

jest.mock('readline');
const mockCreateInterface = jest.mocked(readline.createInterface);
Expand Down
2 changes: 1 addition & 1 deletion packages/cspell/src/application.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { TraceOptions } from '.';
import * as App from './application';
import { LinterOptions } from './options';
import { InMemoryReporter } from './util/InMemoryReporter';
import { asyncIterableToArray } from './util/util';
import { asyncIterableToArray } from './util/async';

const getStdinResult = {
value: '',
Expand Down
7 changes: 4 additions & 3 deletions packages/cspell/src/application.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ import type { CSpellReporter, RunResult } from '@cspell/cspell-types';
import * as cspell from 'cspell-lib';
import { CheckTextInfo, TraceResult, traceWordsAsync, suggestionsForWords, SuggestionError } from 'cspell-lib';
import * as path from 'path';
import { calcFinalConfigInfo, readConfig, readFile } from './fileHelper';
import { calcFinalConfigInfo, readConfig, readFile } from './util/fileHelper';
import { LintRequest, runLint } from './lint';
import { BaseOptions, fixLegacy, LegacyOptions, LinterOptions, SuggestionOptions, TraceOptions } from './options';
import { readStdin } from './util/stdin';
import * as util from './util/util';
import * as async from './util/async';
export { IncludeExcludeFlag } from 'cspell-lib';
export type { TraceResult } from 'cspell-lib';

Expand All @@ -20,7 +21,7 @@ export function lint(fileGlobs: string[], options: LinterOptions, emitters: CSpe

export async function* trace(words: string[], options: TraceOptions): AsyncIterableIterator<TraceResult[]> {
options = fixLegacy(options);
const iWords = options.stdin ? util.mergeAsyncIterables(words, readStdin()) : words;
const iWords = options.stdin ? async.mergeAsyncIterables(words, readStdin()) : words;
const { languageId, locale, allowCompoundWords, ignoreCase } = options;
const configFile = await readConfig(options.config, undefined);
const config = cspell.mergeSettings(cspell.getDefaultSettings(), cspell.getGlobalSettings(), configFile.config);
Expand All @@ -47,7 +48,7 @@ export async function* suggestions(
): AsyncIterable<cspell.SuggestionsForWordResult> {
options = fixLegacy(options);
const configFile = await readConfig(options.config, undefined);
const iWords = options.useStdin ? util.mergeAsyncIterables(words, readStdin()) : words;
const iWords = options.useStdin ? async.mergeAsyncIterables(words, readStdin()) : words;
try {
const results = suggestionsForWords(iWords, options, configFile.config);
yield* results;
Expand Down
152 changes: 98 additions & 54 deletions packages/cspell/src/lint/lint.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
readConfig,
readFileInfo,
readFileListFiles,
} from '../fileHelper';
} from '../util/fileHelper';
import type { CSpellLintResultCache } from '../util/cache';
import { calcCacheSettings, createCache, CreateCacheSettings } from '../util/cache';
import { toApplicationError, toError } from '../util/errors';
Expand All @@ -25,11 +25,11 @@ import { buildGlobMatcher, extractGlobsFromMatcher, extractPatterns, normalizeGl
import { loadReporters, mergeReporters } from '../util/reporters';
import { getTimeMeasurer } from '../util/timer';
import * as util from '../util/util';
import { pipeAsync, isAsyncIterable, filter, pipeSync } from '../util/async';
import { LintRequest } from './LintRequest';

export async function runLint(cfg: LintRequest): Promise<RunResult> {
let { reporter } = cfg;
const { fileLists } = cfg;
cspell.setLogger(getLoggerFromReporter(reporter));
const configErrors = new Set<string>();

Expand Down Expand Up @@ -121,11 +121,11 @@ export async function runLint(cfg: LintRequest): Promise<RunResult> {
}

async function processFiles(
files: string[],
files: string[] | AsyncIterable<string>,
configInfo: ConfigInfo,
cacheSettings: CreateCacheSettings
): Promise<RunResult> {
const fileCount = files.length;
const fileCount = files instanceof Array ? files.length : undefined;
const status: RunResult = runResult();
const cache = createCache(cacheSettings);
const failFast = cfg.options.failFast ?? configInfo.config.failFast ?? false;
Expand All @@ -134,7 +134,7 @@ export async function runLint(cfg: LintRequest): Promise<RunResult> {
reporter.progress({
type: 'ProgressFileComplete',
fileNum,
fileCount,
fileCount: fileCount ?? fileNum,
filename,
elapsedTimeMs: result?.elapsedTimeMs,
processed: result?.processed,
Expand All @@ -143,10 +143,11 @@ export async function runLint(cfg: LintRequest): Promise<RunResult> {
});

async function* loadAndProcessFiles() {
for (let i = 0; i < files.length; i++) {
const filename = files[i];
let i = 0;
for await (const filename of files) {
++i;
const result = await processFile(filename, configInfo, cache);
yield { filename, fileNum: i + 1, result };
yield { filename, fileNum: i, result };
}
}

Expand Down Expand Up @@ -222,19 +223,9 @@ export async function runLint(cfg: LintRequest): Promise<RunResult> {
reporter = mergeReporters(cfg.reporter, ...loadReporters(configInfo.config));
cspell.setLogger(getLoggerFromReporter(reporter));

const useGitignore = cfg.options.gitignore ?? configInfo.config.useGitignore ?? false;
const gitignoreRoots = cfg.options.gitignoreRoot ?? configInfo.config.gitignoreRoot;
const gitIgnore = useGitignore ? await generateGitIgnore(gitignoreRoots) : undefined;

const cliGlobs: Glob[] = cfg.fileGlobs;
const allGlobs: Glob[] = cliGlobs.length ? cliGlobs : configInfo.config.files || [];
const combinedGlobs = normalizeGlobsToRoot(allGlobs, cfg.root, false);
const cliExcludeGlobs = extractPatterns(cfg.excludes).map((p) => p.glob);
const normalizedExcludes = normalizeGlobsToRoot(cliExcludeGlobs, cfg.root, true);
const includeGlobs = combinedGlobs.filter((g) => !g.startsWith('!'));
const excludeGlobs = combinedGlobs.filter((g) => g.startsWith('!')).concat(normalizedExcludes);
const fileGlobs: string[] = includeGlobs;
const hasFileLists = !!fileLists.length;
const globInfo = await determineGlobs(configInfo, cfg);
const { fileGlobs, excludeGlobs } = globInfo;
const hasFileLists = !!cfg.fileLists.length;
if (!fileGlobs.length && !hasFileLists) {
// Nothing to do.
return runResult();
Expand All @@ -248,28 +239,10 @@ export async function runLint(cfg: LintRequest): Promise<RunResult> {

// Get Exclusions from the config files.
const { root } = cfg;
const globsToExclude = (configInfo.config.ignorePaths || []).concat(excludeGlobs);
const globMatcher = buildGlobMatcher(globsToExclude, root, true);
const ignoreGlobs = extractGlobsFromMatcher(globMatcher);
// cspell:word nodir
const globOptions: GlobOptions = {
root,
cwd: root,
ignore: ignoreGlobs.concat(normalizedExcludes),
nodir: true,
};
const enableGlobDot = cfg.enableGlobDot ?? configInfo.config.enableGlobDot;
if (enableGlobDot !== undefined) {
globOptions.dot = enableGlobDot;
}

try {
const cacheSettings = await calcCacheSettings(configInfo.config, cfg.options, root);
const foundFiles = await (hasFileLists
? useFileLists(fileLists, allGlobs, root, enableGlobDot)
: findFiles(fileGlobs, globOptions));
const filtered = gitIgnore ? await gitIgnore.filterOutIgnored(foundFiles) : foundFiles;
const files = filterFiles(filtered, globMatcher);
const files = await determineFilesToCheck(configInfo, cfg, reporter, globInfo);

return await processFiles(files, configInfo, cacheSettings);
} catch (e) {
Expand Down Expand Up @@ -297,6 +270,76 @@ Options:
MessageTypes.Info
);
}
}

interface AppGlobInfo {
/** globs from cli or config.files */
allGlobs: Glob[];
/** GitIgnore config to use. */
gitIgnore: GitIgnore | undefined;
/** file globs used to search for matching files. */
fileGlobs: string[];
/** globs to exclude files found. */
excludeGlobs: string[];
/** normalized cli exclude globs */
normalizedExcludes: string[];
}

async function determineGlobs(configInfo: ConfigInfo, cfg: LintRequest): Promise<AppGlobInfo> {
const useGitignore = cfg.options.gitignore ?? configInfo.config.useGitignore ?? false;
const gitignoreRoots = cfg.options.gitignoreRoot ?? configInfo.config.gitignoreRoot;
const gitIgnore = useGitignore ? await generateGitIgnore(gitignoreRoots) : undefined;

const cliGlobs: Glob[] = cfg.fileGlobs;
const allGlobs: Glob[] = cliGlobs.length ? cliGlobs : configInfo.config.files || [];
const combinedGlobs = normalizeGlobsToRoot(allGlobs, cfg.root, false);
const cliExcludeGlobs = extractPatterns(cfg.excludes).map((p) => p.glob);
const normalizedExcludes = normalizeGlobsToRoot(cliExcludeGlobs, cfg.root, true);
const includeGlobs = combinedGlobs.filter((g) => !g.startsWith('!'));
const excludeGlobs = combinedGlobs.filter((g) => g.startsWith('!')).concat(normalizedExcludes);
const fileGlobs: string[] = includeGlobs;

return { allGlobs, gitIgnore, fileGlobs, excludeGlobs, normalizedExcludes };
}

async function determineFilesToCheck(
configInfo: ConfigInfo,
cfg: LintRequest,
reporter: CSpellReporter,
globInfo: AppGlobInfo
): Promise<string[] | AsyncIterable<string>> {
async function _determineFilesToCheck(): Promise<string[] | AsyncIterable<string>> {
const { fileLists } = cfg;
const hasFileLists = !!fileLists.length;
const { allGlobs, gitIgnore, fileGlobs, excludeGlobs, normalizedExcludes } = globInfo;

// Get Exclusions from the config files.
const { root } = cfg;
const globsToExclude = (configInfo.config.ignorePaths || []).concat(excludeGlobs);
const globMatcher = buildGlobMatcher(globsToExclude, root, true);
const ignoreGlobs = extractGlobsFromMatcher(globMatcher);
// cspell:word nodir
const globOptions: GlobOptions = {
root,
cwd: root,
ignore: ignoreGlobs.concat(normalizedExcludes),
nodir: true,
};
const enableGlobDot = cfg.enableGlobDot ?? configInfo.config.enableGlobDot;
if (enableGlobDot !== undefined) {
globOptions.dot = enableGlobDot;
}

const filterFiles = filter(filterFilesFn(globMatcher));
const foundFiles = await (hasFileLists
? useFileLists(fileLists, allGlobs, root, enableGlobDot)
: findFiles(fileGlobs, globOptions));
const filtered = gitIgnore ? await gitIgnore.filterOutIgnored(foundFiles) : foundFiles;
const files = isAsyncIterable(filtered)
? pipeAsync(filtered, filterFiles)
: [...pipeSync(filtered, filterFiles)];
return files;
}

function isExcluded(filename: string, globMatcherExclude: GlobMatcher) {
if (cspell.isBinaryFile(URI.file(filename))) {
Expand All @@ -317,24 +360,17 @@ Options:
return r.matched;
}

function extractGlobSource(g: GlobPatternWithRoot | GlobPatternNormalized) {
const { glob, rawGlob, source } = <GlobPatternNormalized>g;
return {
glob: rawGlob || glob,
source,
};
}

function filterFiles(files: string[], globMatcherExclude: GlobMatcher): string[] {
function filterFilesFn(globMatcherExclude: GlobMatcher): (file: string) => boolean {
const patterns = globMatcherExclude.patterns;
const excludeInfo = patterns
.map(extractGlobSource)
.map(({ glob, source }) => `Glob: ${glob} from ${source}`)
.filter(util.uniqueFn());
reporter.info(`Exclusion Globs: \n ${excludeInfo.join('\n ')}\n`, MessageTypes.Info);
const result = files.filter(util.uniqueFn()).filter((filename) => !isExcluded(filename, globMatcherExclude));
return result;
return (filename: string): boolean => !isExcluded(filename, globMatcherExclude);
}

return _determineFilesToCheck();
}

function extractContext(
Expand Down Expand Up @@ -374,6 +410,14 @@ function extractContext(
return context;
}

function extractGlobSource(g: GlobPatternWithRoot | GlobPatternNormalized) {
const { glob, rawGlob, source } = <GlobPatternNormalized>g;
return {
glob: rawGlob || glob,
source,
};
}

function runResult(init: Partial<RunResult> = {}): RunResult {
const { files = 0, filesWithIssues = new Set<string>(), issues = 0, errors = 0, cachedFiles = 0 } = init;
return { files, filesWithIssues, issues, errors, cachedFiles };
Expand Down Expand Up @@ -421,7 +465,7 @@ async function useFileLists(
includeGlobPatterns: Glob[],
root: string,
dot: boolean | undefined
): Promise<string[]> {
): Promise<string[] | AsyncIterable<string>> {
includeGlobPatterns = includeGlobPatterns.length ? includeGlobPatterns : ['**'];
const options: GlobMatchOptions = { root, mode: 'include' };
if (dot !== undefined) {
Expand All @@ -430,6 +474,6 @@ async function useFileLists(
const globMatcher = new GlobMatcher(includeGlobPatterns, options);

const files = await readFileListFiles(fileListFiles);

return files.filter((file) => globMatcher.match(file));
const filterFiles = (file: string) => globMatcher.match(file);
return files instanceof Array ? files.filter(filterFiles) : pipeAsync(files, filter(filterFiles));
}
Loading

0 comments on commit 8a03b8a

Please sign in to comment.