diff --git a/packages/core/README.md b/packages/core/README.md index 74da6f0b..f4a82347 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -1,11 +1,53 @@ # `@jscpd/core` -> TODO: description +> core package for detect duplicates, depends only on eventemmiter3. -## Usage +## Installation ``` -const core = require('@jscpd/core'); +npm install @jscpd/core --save +``` + +## Usage + +```typescript +import {Tokenizer} from '@jscpd/tokenizer'; +import { + Detector, + MemoryStore, + IOptions, + IClone, + IStore, + ITokenizer +} from '@jscpd/core'; + +const options: IOptions = { + minLines: 5, + maxLines: 500, +} + +const tokenizer: ITokenizer = new Tokenizer(); + +// here you can use any store what implement IStore interface +const store: IStore = new MemoryStore(); + +// list of validators, implemented IValidator interface, validate clones +const validators = []; + +const detector = new Detector(tokenizer, store, validators, options); + +( async () => { + const format = 'javascript'; + const code: string = '...string with code...'; + const clones: IClone[] = await detector.detect('source_id', code, format); + + console.log(clones); +})(); -// TODO: DEMONSTRATE API ``` + +![ga tracker](https://www.google-analytics.com/collect?v=1&a=257770996&t=pageview&dl=https%3A%2F%2Fgithub.com%2Fkucherenko%2Fjscpd&ul=en-us&de=UTF-8&cid=978224512.1377738459&tid=UA-730549-17&z=887657232 "ga tracker") + +## License + +[MIT](LICENSE) © Andrey Kucherenko diff --git a/packages/core/src/detector.ts b/packages/core/src/detector.ts index 56f4b60e..91b53d70 100644 --- a/packages/core/src/detector.ts +++ b/packages/core/src/detector.ts @@ -1,20 +1,18 @@ import {RabinKarp} from './rabin-karp'; -import {IClone, ICloneValidator, IOptions, IStore, ITokenizer, ITokensMap} from './interfaces'; +import {IClone, ICloneValidator, IMapFrame, IOptions, IStore, ITokenizer, ITokensMap} from './interfaces'; import {LinesLengthCloneValidator} from './validators'; +// TODO replace to own event emitter import EventEmitter = require('eventemitter3'); - export type DetectorEvents = 'CLONE_FOUND' | 'CLONE_SKIPPED' | 'START_DETECTION'; - export class Detector extends EventEmitter { private algorithm: RabinKarp; constructor( private tokenizer: ITokenizer, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private readonly store: IStore, + private readonly store: IStore, private cloneValidators: ICloneValidator[] = [], private readonly options: IOptions) { super(); diff --git a/packages/finder/README.md b/packages/finder/README.md index b0e97019..ce029178 100644 --- a/packages/finder/README.md +++ b/packages/finder/README.md @@ -1,11 +1,49 @@ # `@jscpd/finder` -> TODO: description +> core package for detect duplicates, depends only on eventemmiter3. -## Usage +## Installation ``` -const core = require('@jscpd/finder'); +npm install @jscpd/finder --save +``` + +## Usage + +```typescript +import {Tokenizer} from '@jscpd/tokenizer'; +import { + MemoryStore, + IOptions, + IClone, + IStore, + ITokenizer +} from '@jscpd/core'; +import {EntryWithContent, getFilesToDetect, InFilesDetector} from '@jscpd/finder'; + +const options: IOptions = { + minLines: 5, + maxLines: 500, + path: ['list of folders and files to analyse for clones'] +} + +const tokenizer: ITokenizer = new Tokenizer(); +// here you can use any store what implement IStore interface +const store: IStore = new MemoryStore(); +const statistic = new Statistic(options); -// TODO: DEMONSTRATE API +const files: EntryWithContent[] = getFilesToDetect(options); + +const detector = new InFilesDetector(tokenizer, store, statistic, options); + +( async () => { + const clones: IClone[] = await detector.detect(files); +})(); ``` + + +![ga tracker](https://www.google-analytics.com/collect?v=1&a=257770996&t=pageview&dl=https%3A%2F%2Fgithub.com%2Fkucherenko%2Fjscpd&ul=en-us&de=UTF-8&cid=978224512.1377738459&tid=UA-730549-17&z=887657232 "ga tracker") + +## License + +[MIT](LICENSE) © Andrey Kucherenko diff --git a/packages/jscpd/README.md b/packages/jscpd/README.md index 6c08503a..35c5bf8e 100644 --- a/packages/jscpd/README.md +++ b/packages/jscpd/README.md @@ -34,6 +34,7 @@ The jscpd tool implements [Rabin-Karp](https://en.wikipedia.org/wiki/Rabin%E2%80 - [Badge](#badge) - [PMD CPD XML](#pmd-cpd-xml) - [JSON](#json-reporters) +- [API](#api) - [Changelog](#changelog) - [Who uses jscpd](#who-uses-jscpd) - [Contributors](#contributors) @@ -304,14 +305,13 @@ import {UserService} from './services'; --> ``` -## JSCPD Reporters +## Reporters ### HTML [Demo report](http://kucherenko.github.io/jscpd-report.html) ### Badge - -![jscpd](assets/jscpd-badge.svg) +![jscpd](../../assets/jscpd-badge.svg) More info [jscpd-badge-reporter](https://github.com/kucherenko/jscpd-badge-reporter) ### PMD CPD XML @@ -398,59 +398,27 @@ More info [jscpd-badge-reporter](https://github.com/kucherenko/jscpd-badge-repor "percentage": 45.33, "newDuplicatedLines": 0, "newClones": 0 - }, - "threshold": 10 + } } } ``` ## API +For run cli version use following code: ```typescript -import { - JSCPD, - IClone, - IOptions, - MATCH_SOURCE_EVENT, - CLONE_FOUND_EVENT, - SOURCE_SKIPPED_EVENT, - END_EVENT -} from 'jscpd'; - -const options: IOptions = {}; - -const cpd = new JSCPD(options); - -const code = '...string with my code...'; - -cpd.on(MATCH_SOURCE_EVENT, (source) => { - // new source detection started - console.log(source); -}); +import {IClone} from '@jscpd/core'; +import {jscpd} from 'jscpd'; -cpd.on(CLONE_FOUND_EVENT, (clone: IClone) => { - // clone found event - console.log(clone); -}); - -cpd.on(SOURCE_SKIPPED_EVENT, (stat) => { - // skipped source due size (see max-size, min-lines and max-lines options) - console.log(stat); -}); - -cpd.on(END_EVENT, (clones: IClone[]) => { - // detection process finished - console.log(clones); -}); - -cpd.detect(code, { id: 'test', format: 'markup' }) - .then((clones: IClone[]) => console.log(clones)); - - -cpd.detectInFiles(['./src', './tests']) - .then((clones: IClone[]) => console.log(clones)); +const clones: Promise = jscpd(process.argv); +(async () => { + const clones: IClone[] = await jscpd(['', '', '/path/to/file/or/folder', '-m', 'weak', '--silent']); +})(); ``` +If you are going to detect clones in file system you can use [@jscpd/finder](../finder) for make a powerful detector. +In case of detect clones in browser or not node.js environment you can build you own solution base on [@jscpd/code](../core) + ## Changelog [Changelog](CHANGELOG.md) diff --git a/packages/leveldb-store/README.md b/packages/leveldb-store/README.md index 74da6f0b..c38f2de1 100644 --- a/packages/leveldb-store/README.md +++ b/packages/leveldb-store/README.md @@ -1,11 +1,41 @@ -# `@jscpd/core` +# `@jscpd/leveldb-store` -> TODO: description +> store for jscpd, used for large repositories + +## Installation + +``` +npm install @jscpd/leveldb-store --save +``` ## Usage ``` -const core = require('@jscpd/core'); +import {Tokenizer} from '@jscpd/tokenizer'; +import { + Detector, + IOptions, + IClone, + IStore, + ITokenizer +} from '@jscpd/core'; +import LeveldbStore from '@jscpd/leveldb-store'; + +const options: IOptions = { + minLines: 5, + maxLines: 500, +} + +const tokenizer: ITokenizer = new Tokenizer(); + +const store: IStore = new LeveldbStore(); + +const detector = new Detector(tokenizer, store, [], options); -// TODO: DEMONSTRATE API ``` + +![ga tracker](https://www.google-analytics.com/collect?v=1&a=257770996&t=pageview&dl=https%3A%2F%2Fgithub.com%2Fkucherenko%2Fjscpd&ul=en-us&de=UTF-8&cid=978224512.1377738459&tid=UA-730549-17&z=887657232 "ga tracker") + +## License + +[MIT](LICENSE) © Andrey Kucherenko diff --git a/packages/redis-store/README.md b/packages/redis-store/README.md index 74da6f0b..9fb7de7b 100644 --- a/packages/redis-store/README.md +++ b/packages/redis-store/README.md @@ -1,11 +1,43 @@ -# `@jscpd/core` +# `@jscpd/leveldb-store` -> TODO: description +**!NOT implemented yet** + +> store for jscpd, used [redis](https://redis.io/) data store for hashes + +## Installation + +``` +npm install @jscpd/redis-store --save +``` ## Usage ``` -const core = require('@jscpd/core'); +import {Tokenizer} from '@jscpd/tokenizer'; +import { + Detector, + IOptions, + IClone, + IStore, + ITokenizer +} from '@jscpd/core'; +import RedisdbStore from '@jscpd/redis-store'; + +const options: IOptions = { + minLines: 5, + maxLines: 500, +} + +const tokenizer: ITokenizer = new Tokenizer(); + +const store: IStore = new RedisdbStore(); + +const detector = new Detector(tokenizer, store, [], options); -// TODO: DEMONSTRATE API ``` + +![ga tracker](https://www.google-analytics.com/collect?v=1&a=257770996&t=pageview&dl=https%3A%2F%2Fgithub.com%2Fkucherenko%2Fjscpd&ul=en-us&de=UTF-8&cid=978224512.1377738459&tid=UA-730549-17&z=887657232 "ga tracker") + +## License + +[MIT](LICENSE) © Andrey Kucherenko diff --git a/packages/tokenizer/README.md b/packages/tokenizer/README.md index 41d1cfe6..33c4ba28 100644 --- a/packages/tokenizer/README.md +++ b/packages/tokenizer/README.md @@ -1,11 +1,29 @@ # `@jscpd/tokenizer` -> TODO: description +> tokenizer is package from @jscpd used for convert programming code to list of tokens + + +## Installation + +``` +npm install @jscpd/tokenizer --save +``` ## Usage ``` -const core = require('@jscpd/tokenizer'); +import {IOptions, ITokensMap} from '@jscpd/core'; +import {Tokenizer} from '@jscpd/tokenizer'; + +const tokenizer = new Tokenizer(); +const options: IOptions = {}; + +const maps: ITokensMap[] = tokenizer.generateMaps('source_id', 'let a = "11"', 'javascript', options); -// TODO: DEMONSTRATE API ``` + +![ga tracker](https://www.google-analytics.com/collect?v=1&a=257770996&t=pageview&dl=https%3A%2F%2Fgithub.com%2Fkucherenko%2Fjscpd&ul=en-us&de=UTF-8&cid=978224512.1377738459&tid=UA-730549-17&z=887657232 "ga tracker") + +## License + +[MIT](LICENSE) © Andrey Kucherenko