diff --git a/examples/api/example-skipIsolated.ts b/examples/api/example-skipIsolated.ts new file mode 100644 index 00000000..4d4eb6a0 --- /dev/null +++ b/examples/api/example-skipIsolated.ts @@ -0,0 +1,14 @@ +import {detectClones} from "jscpd"; + +(async () => { + const clones = await detectClones({ + path: [ + __dirname + '/../fixtures' + ], + skipIsolated: [ + ['packages/businessA', 'packages/businessB', 'packages/businessC'], + ], + silent: true + }); + console.log(clones); +})() diff --git a/fixtures/lib1/file2.c b/fixtures/lib1/file2.c new file mode 100644 index 00000000..25e333b1 --- /dev/null +++ b/fixtures/lib1/file2.c @@ -0,0 +1,29 @@ +/* + * Copy the size of snapshot frame "sn" to frame "fr". Do the same for all + * following frames and children. + * Returns a pointer to the old current window, or NULL. + */ +static win_T *restore_snapshot_rec(frame_T *sn, frame_T *fr) +{ + win_T *wp = NULL; + win_T *wp2; + + fr->fr_height = sn->fr_height; + fr->fr_width = sn->fr_width; + if (fr->fr_layout == FR_LEAF) { + frame_new_height(fr, fr->fr_height, FALSE, FALSE); + frame_new_width(fr, fr->fr_width, FALSE, FALSE); + wp = sn->fr_win; + } + win_T *wp = NULL; + win_T *wp2; + + fr->fr_height = sn->fr_height; + fr->fr_width = sn->fr_width; + if (fr->fr_layout == FR_LEAF) { + frame_new_height(fr, fr->fr_height, FALSE, FALSE); + frame_new_width(fr, fr->fr_width, FALSE, FALSE); + wp = sn->fr_win; + } + return wp; +} diff --git a/fixtures/lib1/file_1.js b/fixtures/lib1/file_1.js new file mode 100644 index 00000000..91afd38e --- /dev/null +++ b/fixtures/lib1/file_1.js @@ -0,0 +1,55 @@ +/** +*12312 +*/ +function utf8_encode ( str_data ) { + // Encodes an ISO-8859-1 string to UTF-8 + // + // + original by: Webtoolkit.info (http://www.webtoolkit.info/) + str_data = str_data.replace(/\r\n/g,"\n"); + var utftext = ""; + + for (var n = 0; n < str_data.length; n++) { + var c = str_data.charCodeAt(n); + if (c < 128) { + utftext += String.fromCharCode(c); + } else if((c > 127) && (c < 2048)) { + utftext += String.fromCharCode((c >> 6) | 192); + utftext += String.fromCharCode((c & 63) | 128); + } else { + utftext += String.fromCharCode((c >> 12) | 224); + utftext += String.fromCharCode(((c >> 6) & 63) | 128); + utftext += String.fromCharCode((c & 63) | 128); + } + } + + return utftext; +} + +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; \ No newline at end of file diff --git a/fixtures/lib2/file_2.js b/fixtures/lib2/file_2.js new file mode 100644 index 00000000..c64e76cf --- /dev/null +++ b/fixtures/lib2/file_2.js @@ -0,0 +1,56 @@ +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; \ No newline at end of file diff --git a/packages/core/src/interfaces/options.interface.ts b/packages/core/src/interfaces/options.interface.ts index 91cf9ed2..bd9a55f4 100644 --- a/packages/core/src/interfaces/options.interface.ts +++ b/packages/core/src/interfaces/options.interface.ts @@ -27,6 +27,7 @@ export interface IOptions { absolute?: boolean; noSymlinks?: boolean; skipLocal?: boolean; + skipIsolated?: string[][]; ignoreCase?: boolean; gitignore?: boolean; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/finder/src/in-files-detector.ts b/packages/finder/src/in-files-detector.ts index 7277c005..02b50694 100644 --- a/packages/finder/src/in-files-detector.ts +++ b/packages/finder/src/in-files-detector.ts @@ -13,7 +13,7 @@ import { } from '@jscpd/core'; import {getFormatByFile} from '@jscpd/tokenizer'; import {EntryWithContent, IHook, IReporter} from './interfaces'; -import {SkipLocalValidator} from './validators'; +import {SkipLocalValidator, SkipIsolatedValidator} from './validators'; export class InFilesDetector { @@ -55,6 +55,10 @@ export class InFilesDetector { validators.push(new SkipLocalValidator()); } + if (options.skipIsolated) { + validators.push(new SkipIsolatedValidator()); + } + const detector = new Detector(this.tokenizer, store, validators, options); this.subscribes.forEach((listener: ISubscriber) => { diff --git a/packages/finder/src/validators/index.ts b/packages/finder/src/validators/index.ts index 634a197f..2f00458d 100644 --- a/packages/finder/src/validators/index.ts +++ b/packages/finder/src/validators/index.ts @@ -1 +1,2 @@ export * from './skip-local.validator'; +export * from './skip-isolated.validator'; diff --git a/packages/finder/src/validators/skip-isolated.validator.ts b/packages/finder/src/validators/skip-isolated.validator.ts new file mode 100644 index 00000000..2d302723 --- /dev/null +++ b/packages/finder/src/validators/skip-isolated.validator.ts @@ -0,0 +1,46 @@ +import {getOption, IClone, ICloneValidator, IOptions, IValidationResult} from '@jscpd/core'; +import {isAbsolute, relative} from "path"; + +export class SkipIsolatedValidator implements ICloneValidator { + isRelativeMemoMap = new Map(); + + + validate(clone: IClone, options: IOptions): IValidationResult { + const status = !this.shouldSkipClone(clone, options); + return { + status, + clone, + message: [ + `Sources of duplication located in isolated folder (${clone.duplicationA.sourceId}, ${clone.duplicationB.sourceId})` + ] + }; + } + + public shouldSkipClone(clone: IClone, options: IOptions): boolean { + const skipIsolatedPathList: string[][] = getOption('skipIsolated', options); + return skipIsolatedPathList.some( + (dirList) => { + const relA = dirList.find(dir => this.isRelativeMemo(clone.duplicationA.sourceId, dir)); + if (!relA) { + return false; + } + const relB = dirList.find(dir => this.isRelativeMemo(clone.duplicationB.sourceId, dir)); + return relB && relA !== relB; + } + ); + } + + private isRelativeMemo(file: string, dir: string) { + const memoKey = `${file},${dir}`; + if (this.isRelativeMemoMap.has(memoKey)) return this.isRelativeMemoMap.get(memoKey); + const isRel = SkipIsolatedValidator.isRelative(file, dir) + this.isRelativeMemoMap.set(memoKey, isRel); + return isRel; + } + + private static isRelative(file: string, path: string): boolean { + const rel = relative(path, file); + return rel !== '' && !rel.startsWith('..') && !isAbsolute(rel); + } + +} diff --git a/packages/jscpd/README.md b/packages/jscpd/README.md index 01e7e7fe..ac7e7415 100644 --- a/packages/jscpd/README.md +++ b/packages/jscpd/README.md @@ -92,9 +92,9 @@ Minimal block size of code in tokens. The block of code less than `min-tokens` w - Cli options: `--min-tokens`, `-k` - Type: **number** - Default: **50** - + *This option is called ``minTokens`` in the config file.* - + ### Min Lines Minimal block size of code in lines. The block of code less than `min-lines` will be skipped. @@ -243,6 +243,31 @@ will detect clones in separate folders only, clones from same folder will be ski - Type: **boolean** - Default: **false** +### Skip Isolated +Use for skip detect duplications in isolated folder. + +Example: +```bash +jscpd . --skipLocal "packages/businessA|packages/businessB,libs/businessA|libs|businessB" +``` +clones from the isolated folder will be skipped. + +```text +monorepo/ +├─ .monorepo.config.json +├─ node_modules/ +├─ packages/ +│ ├─ businessA/ // for TEAM A only +│ ├─ businessB/ // for TEAM B only +├─ globals/ +├─ infra / +``` + +- Cli options: `--skipIsolated` +- Type: **string[][]** + + + ### Formats Extensions Define the list of formats with file extensions. Available over [150 formats](../../supported_formats.md). diff --git a/packages/jscpd/__tests__/options.test.ts b/packages/jscpd/__tests__/options.test.ts index 890ce658..cf309d81 100644 --- a/packages/jscpd/__tests__/options.test.ts +++ b/packages/jscpd/__tests__/options.test.ts @@ -126,6 +126,49 @@ describe('jscpd options', () => { }); }); + describe('skip isolated', () => { + const folder1Path = pathToFixtures + '/folder1'; + const folder2Path = pathToFixtures + '/folder2'; + const lib1Path = pathToFixtures + '/lib1'; + const lib2Path = pathToFixtures + '/lib2'; + + it('should not skip clone if it is located in isolated folder without --skipIsolated option', async () => { + const clones: IClone[] = await jscpd([ + '', '', + folder1Path, + folder2Path, + lib1Path, + lib2Path, + ]); + // lib2 file_2.js lib2 file_2.js + // lib1 file_1.js lib2 file_2.js + // lib1 file2.c lib1 file2.c + // folder2 file_2.js lib2 file_2.js + // folder1 file_1.js lib1 file_1.js + // folder1 file2.c lib1 file2.c + expect(clones.length).to.equal(6); + }); + + it('should skip clone if it is located in isolated folder with --skipIsolated option', async () => { + const clones: IClone[] = await jscpd([ + '', '', + folder1Path, + folder2Path, + lib1Path, + lib2Path, + '--skipIsolated', + // folder1Path is isolated with folder2Path, lib1Path is isolated with lib2Path + `${folder1Path}|${folder2Path},${lib1Path}|${lib2Path}` + ]); + // lib2 file_2.js lib2 file_2.js + // lib1 file2.c lib1 file2.c + // folder2 file_2.js lib2 file_2.js + // folder1 file_1.js lib1 file_1.js + // folder1 file2.c lib1 file2.c + expect(clones.length).to.equal(5); + }); + }); + describe('silent', () => { it('should not print more information about detection process', async () => { await jscpd(['', '', fileWithClones, '--silent']); diff --git a/packages/jscpd/src/init/cli.ts b/packages/jscpd/src/init/cli.ts index b92bc2c5..2a98e188 100644 --- a/packages/jscpd/src/init/cli.ts +++ b/packages/jscpd/src/init/cli.ts @@ -50,6 +50,7 @@ export function initCli(packageJson, argv: string[]): Command { .option('-v, --verbose', 'show full information during detection process') .option('--list', 'show list of total supported formats') .option('--skipLocal', 'skip duplicates in local folders, just detect cross folders duplications') + .option('--skipIsolated [string]', 'skip duplicates cross from isolated folders for monorepo (e.g. packages/a|packages/b,lib/a|lib/b)') .option('--exitCode [number]', 'exit code to use when code duplications are detected') cli.parse(argv); diff --git a/packages/jscpd/src/options.ts b/packages/jscpd/src/options.ts index ef7a2bb9..f5357b08 100644 --- a/packages/jscpd/src/options.ts +++ b/packages/jscpd/src/options.ts @@ -27,6 +27,7 @@ const convertCliToOptions = (cli: Command): Partial => { absolute: cli.absolute, noSymlinks: cli.noSymlinks, skipLocal: cli.skipLocal, + skipIsolated: cli.skipIsolated?.split(',')?.map(s => s.split('|')), ignoreCase: cli.ignoreCase, gitignore: cli.gitignore, exitCode: cli.exitCode,