Skip to content

Commit

Permalink
fix(core): ignored directories should never contain projects (#15242)
Browse files Browse the repository at this point in the history
  • Loading branch information
AgentEnder committed Feb 24, 2023
1 parent 2f09cf4 commit a815be5
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 91 deletions.
2 changes: 1 addition & 1 deletion packages/eslint-plugin-nx/src/utils/runtime-lint-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
workspaceRoot,
} from '@nrwl/devkit';
import { getPath, pathExists } from './graph-utils';
import { readFileIfExisting } from 'nx/src/project-graph/file-utils';
import { readFileIfExisting } from 'nx/src/utils/fileutils';
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
import {
findProjectForPath,
Expand Down
11 changes: 0 additions & 11 deletions packages/nx/src/command-line/dep-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -436,17 +436,6 @@ let currentDepGraphClientResponse: ProjectGraphClientResponse = {
exclude: [],
};

function getIgnoredGlobs(root: string) {
const ig = ignore();
try {
ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
} catch {}
try {
ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
} catch {}
return ig;
}

function debounce(fn: (...args) => void, time: number) {
let timeout: NodeJS.Timeout;

Expand Down
32 changes: 32 additions & 0 deletions packages/nx/src/config/workspaces.glob-for-project-files.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { TempFs } from '../utils/testing/temp-fs';
import { globForProjectFiles } from './workspaces';

describe('globForProjectFiles', () => {
let fs: TempFs;
beforeAll(() => {
fs = new TempFs('glob-for-project-files');
});
afterAll(() => {
fs.cleanup();
});

it('should not find files that are listed by .nxignore', async () => {
await fs.createFile('.nxignore', `not-projects`);
await fs.createFile(
'not-projects/project.json',
JSON.stringify({
name: 'not-project-1',
})
);
await fs.createFile(
'projects/project.json',
JSON.stringify({
name: 'project-1',
})
);
expect(globForProjectFiles(fs.tempDir)).not.toContain(
'not-projects/project.json'
);
expect(globForProjectFiles(fs.tempDir)).toContain('projects/project.json');
});
});
38 changes: 19 additions & 19 deletions packages/nx/src/config/workspaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ import {
shouldMergeAngularProjects,
} from '../adapter/angular-json';
import { getNxRequirePaths } from '../utils/installation-directory';
import {
ALWAYS_IGNORE,
getIgnoredGlobs,
getIgnoreObject,
} from '../utils/ignore';

export class Workspaces {
private cachedProjectsConfig: ProjectsConfigurations;
Expand Down Expand Up @@ -561,7 +566,7 @@ function removeRelativePath(pattern: string): string {
}

export function globForProjectFiles(
root,
root: string,
nxJson?: NxJsonConfiguration,
ignorePluginInference = false
) {
Expand Down Expand Up @@ -615,35 +620,30 @@ export function globForProjectFiles(
* .gitignore and .nxignore files below.
*/

const ALWAYS_IGNORE = [
const staticIgnores = [
'node_modules',
'**/node_modules',
'dist',
'.git',
...globsToExclude,
...getIgnoredGlobs(root, false),
];

/**
* TODO: This utility has been implemented multiple times across the Nx codebase,
* discuss whether it should be moved to a shared location.
*/
const ig = ignore();
try {
ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
} catch {}
try {
ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
} catch {}

const globResults = globSync(combinedProjectGlobPattern, {
ignore: ALWAYS_IGNORE,
const opts = {
ignore: staticIgnores,
absolute: false,
cwd: root,
dot: true,
suppressErrors: true,
});
};

const globResults = globSync(combinedProjectGlobPattern, opts);

projectGlobCache = deduplicateProjectFiles(globResults, ig);
projectGlobCache = deduplicateProjectFiles(globResults);

// TODO @vsavkin remove after Nx 16
if (
Expand All @@ -666,15 +666,15 @@ export function globForProjectFiles(
return projectGlobCache;
}

export function deduplicateProjectFiles(
files: string[],
ig?: Ignore
): string[] {
/**
* @description Loops through files and reduces them to 1 file per project.
* @param files Array of files that may represent projects
*/
export function deduplicateProjectFiles(files: string[]): string[] {
const filtered = new Map();
files.forEach((file) => {
const projectFolder = dirname(file);
const projectFile = basename(file);
if (ig?.ignores(file)) return; // file is in .gitignore or .nxignoreb
if (filtered.has(projectFolder) && projectFile !== 'project.json') return;
filtered.set(projectFolder, projectFile);
});
Expand Down
50 changes: 9 additions & 41 deletions packages/nx/src/daemon/server/watcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,57 +7,25 @@
*/
import { workspaceRoot } from '../../utils/workspace-root';
import type { AsyncSubscription, Event } from '@parcel/watcher';
import { readFileSync } from 'fs';
import { join, relative } from 'path';
import { relative } from 'path';
import { FULL_OS_SOCKET_PATH } from '../socket-utils';
import { handleServerProcessTermination } from './shutdown-utils';
import { Server } from 'net';
import ignore from 'ignore';
import { normalizePath } from '../../utils/path';
import {
getAlwaysIgnore,
getIgnoredGlobs,
getIgnoreObject,
} from '../../utils/ignore';
import { platform } from 'os';

const ALWAYS_IGNORE = [
join(workspaceRoot, 'node_modules'),
join(workspaceRoot, '.git'),
FULL_OS_SOCKET_PATH,
];

function getIgnoredGlobs() {
return [
...ALWAYS_IGNORE,
...getIgnoredGlobsFromFile(join(workspaceRoot, '.nxignore')),
...getIgnoredGlobsFromFile(join(workspaceRoot, '.gitignore')),
];
}

function getIgnoredGlobsFromFile(file: string): string[] {
try {
return readFileSync(file, 'utf-8')
.split('\n')
.map((i) => i.trim())
.filter((i) => !!i && !i.startsWith('#'))
.map((i) => (i.startsWith('/') ? join(workspaceRoot, i) : i));
} catch (e) {
return [];
}
}
const ALWAYS_IGNORE = [...getAlwaysIgnore(workspaceRoot), FULL_OS_SOCKET_PATH];

export type FileWatcherCallback = (
err: Error | null,
changeEvents: Event[] | null
) => Promise<void>;

function configureIgnoreObject() {
const ig = ignore();
try {
ig.add(readFileSync(`${workspaceRoot}/.gitignore`, 'utf-8'));
} catch {}
try {
ig.add(readFileSync(`${workspaceRoot}/.nxignore`, 'utf-8'));
} catch {}
return ig;
}

export async function subscribeToOutputsChanges(
cb: FileWatcherCallback
): Promise<AsyncSubscription> {
Expand Down Expand Up @@ -93,7 +61,7 @@ export async function subscribeToWorkspaceChanges(
* executed by packages which do not have its necessary native binaries available.
*/
const watcher = await import('@parcel/watcher');
const ignoreObj = configureIgnoreObject();
const ignoreObj = getIgnoreObject();

return await watcher.subscribe(
workspaceRoot,
Expand Down Expand Up @@ -136,7 +104,7 @@ export async function subscribeToWorkspaceChanges(
cb(null, nonIgnoredEvents);
}
},
watcherOptions(getIgnoredGlobs())
watcherOptions(getIgnoredGlobs(workspaceRoot))
);
}

Expand Down
9 changes: 2 additions & 7 deletions packages/nx/src/hasher/node-based-file-hasher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { FileHasherBase } from './file-hasher-base';
import { stripIndents } from '../utils/strip-indents';
import ignore from 'ignore';
import { normalizePath } from '../utils/path';
import { getIgnoreObject } from '../utils/ignore';

export class NodeBasedFileHasher extends FileHasherBase {
ignoredGlobs = getIgnoredGlobs();
Expand Down Expand Up @@ -65,9 +66,7 @@ export class NodeBasedFileHasher extends FileHasherBase {
}

function getIgnoredGlobs() {
const ig = ignore();
ig.add(readFileIfExisting(`${workspaceRoot}/.gitignore`));
ig.add(readFileIfExisting(`${workspaceRoot}/.nxignore`));
const ig = getIgnoreObject();
ig.add(stripIndents`
node_modules
tmp
Expand All @@ -76,7 +75,3 @@ function getIgnoredGlobs() {
`);
return ig;
}

function readFileIfExisting(path: string) {
return existsSync(path) ? readFileSync(path, 'utf-8') : '';
}
14 changes: 2 additions & 12 deletions packages/nx/src/project-graph/file-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
readProjectsConfigurationFromProjectGraph,
} from './project-graph';
import { toOldFormat } from '../adapter/angular-json';
import { getIgnoreObject } from '../utils/ignore';

export interface Change {
type: string;
Expand Down Expand Up @@ -42,17 +43,6 @@ export function isDeletedFileChange(
return change.type === 'WholeFileDeleted';
}

export function readFileIfExisting(path: string) {
return existsSync(path) ? readFileSync(path, 'utf-8') : '';
}

function getIgnoredGlobs() {
const ig = ignore();
ig.add(readFileIfExisting(`${workspaceRoot}/.gitignore`));
ig.add(readFileIfExisting(`${workspaceRoot}/.nxignore`));
return ig;
}

export function calculateFileChanges(
files: string[],
allWorkspaceFiles: FileData[],
Expand All @@ -61,7 +51,7 @@ export function calculateFileChanges(
f: string,
r: void | string
) => string = defaultReadFileAtRevision,
ignore = getIgnoredGlobs()
ignore = getIgnoreObject()
): FileChange[] {
files = files.filter((f) => !ignore.ignores(f));

Expand Down
5 changes: 5 additions & 0 deletions packages/nx/src/utils/fileutils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
writeFileSync,
mkdirSync,
statSync,
existsSync,
} from 'fs';
import { dirname } from 'path';
import * as tar from 'tar-stream';
Expand Down Expand Up @@ -154,3 +155,7 @@ export async function extractFileFromTarball(
createReadStream(tarballPath).pipe(createGunzip()).pipe(tarExtractStream);
});
}

export function readFileIfExisting(path: string) {
return existsSync(path) ? readFileSync(path, 'utf-8') : '';
}
66 changes: 66 additions & 0 deletions packages/nx/src/utils/ignore.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import { readFileSync } from 'fs-extra';
import ignore from 'ignore';
import { join } from 'path/posix';
import { readFileIfExisting } from './fileutils';
import { workspaceRoot } from './workspace-root';

/**
* An array of glob patterns that should always be ignored.
* Uses path/posix, since fast-glob requires unix paths.
*/
export const ALWAYS_IGNORE = getAlwaysIgnore();

export function getIgnoredGlobs(
root: string = workspaceRoot,
prependRoot: boolean = true
) {
const files = ['.nxignore', '.gitignore'];
if (prependRoot) {
return [
...getAlwaysIgnore(root),
...files.flatMap((f) => getIgnoredGlobsFromFile(join(root, f), root)),
];
} else {
return [
...getAlwaysIgnore(),
...files.flatMap((f) => getIgnoredGlobsFromFile(join(root, f))),
];
}
}

export function getAlwaysIgnore(root?: string) {
const paths = ['node_modules', '**/node_modules', '.git'];
return root ? paths.map((x) => join(root, x)) : paths;
}

export function getIgnoreObject(root: string = workspaceRoot) {
const ig = ignore();
ig.add(readFileIfExisting(`${root}/.gitignore`));
ig.add(readFileIfExisting(`${root}/.nxignore`));
return ig;
}

function getIgnoredGlobsFromFile(file: string, root?: string): string[] {
try {
const results = [];
const contents = readFileSync(file, 'utf-8');
const lines = contents.split('\n');
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) {
continue;
} else if (trimmed.startsWith('/')) {
if (root) {
results.push(join(root, trimmed));
} else {
results.push(join('.', trimmed));
}
} else {
results.push(trimmed);
}
}
return results;
} catch (e) {
return [];
}
}

1 comment on commit a815be5

@vercel
Copy link

@vercel vercel bot commented on a815be5 Feb 24, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

nx-dev – ./

nx-five.vercel.app
nx-dev-nrwl.vercel.app
nx-dev-git-master-nrwl.vercel.app
nx.dev

Please sign in to comment.