feat(result-comparator): generate comparison results

lib/file-client/file-constants.ts

@@ -4,11 +4,17 @@ export const CACHE_LOCATION = './.cache-eslint-remote-tester';
 /** Directory where results are generated in to */
 export const RESULTS_LOCATION = './eslint-remote-tester-results';
 
-/** Directory where comparison results are generated in to */
+/** Directory name where comparison results are generated in to */
 export const RESULTS_COMPARE_DIR = 'comparison-results';
 
 /** Directory name where comparison results are generated in to */
 export const RESULTS_COMPARE_LOCATION = `${RESULTS_LOCATION}/${RESULTS_COMPARE_DIR}`;
 
+/** Cache JSON for previous scan results */
+const RESULT_COMPARISON_CACHE = '.comparison-cache.json';
+
+/** Location for result comparison cache */
+export const RESULTS_COMPARISON_CACHE_LOCATION = `${CACHE_LOCATION}/${RESULT_COMPARISON_CACHE}`;
+
 /** Base URL for repository cloning */
 export const URL = 'https://github.com';

lib/file-client/index.ts

@@ -4,10 +4,13 @@ export {
     prepareResultsDirectory,
     RESULT_TEMPLATE,
 } from './results-writer';
+export { compareResults, writeComparisonResults } from './result-comparator';
 export {
     CACHE_LOCATION,
     RESULTS_LOCATION,
     RESULTS_COMPARE_DIR,
+    RESULTS_COMPARE_LOCATION,
+    RESULTS_COMPARISON_CACHE_LOCATION,
 } from './file-constants';
 export { removeCachedRepository } from './repository-client';
 export { default as ResultsStore } from './results-store';

lib/file-client/result-comparator.ts

@@ -0,0 +1,110 @@
+import fs from 'fs';
+
+import {
+    RESULTS_COMPARISON_CACHE_LOCATION,
+    RESULTS_COMPARE_LOCATION,
+} from './file-constants';
+import {
+    RESULT_PARSER_TO_COMPARE_TEMPLATE,
+    RESULT_PARSER_TO_EXTENSION,
+    Result,
+    ComparisonTypes,
+    ComparisonResults,
+} from './result-templates';
+import config from '@config';
+
+/**
+ * Compare two sets of `Result`s and get diff of changes
+ * - `added`: results which were present in previous scan but not in the current
+ * - `removed`: results which are present in current scan but not in the previous
+ */
+export function compareResults(current: Result[]): ComparisonResults {
+    const added: Result[] = [];
+    const removed: Result[] = [];
+
+    const previous = readCache();
+    const all = [...current, ...previous];
+
+    for (const result of all) {
+        const matcher = (r: Result) => equals(result, r);
+        const isInCurrent = current.find(matcher);
+        const isInPrevious = previous.find(matcher);
+
+        if (isInCurrent && !isInPrevious) {
+            added.push(result);
+        } else if (isInPrevious && !isInCurrent) {
+            removed.push(result);
+        }
+    }
+
+    return { added, removed };
+}
+
+/**
+ * Write comparison results to file system and update cache with current results
+ */
+export function writeComparisonResults(
+    comparisonResults: ComparisonResults,
+    currentScanResults: Result[]
+): void {
+    writeComparisons(comparisonResults);
+    updateCache(currentScanResults);
+}
+
+function readCache(): Result[] {
+    if (!fs.existsSync(RESULTS_COMPARISON_CACHE_LOCATION)) {
+        return [];
+    }
+
+    const cache = fs.readFileSync(RESULTS_COMPARISON_CACHE_LOCATION, 'utf8');
+    return JSON.parse(cache);
+}
+
+function updateCache(currentScanResults: Result[]): void {
+    if (fs.existsSync(RESULTS_COMPARISON_CACHE_LOCATION)) {
+        fs.unlinkSync(RESULTS_COMPARISON_CACHE_LOCATION);
+    }
+
+    fs.writeFileSync(
+        RESULTS_COMPARISON_CACHE_LOCATION,
+        JSON.stringify(currentScanResults),
+        'utf8'
+    );
+}
+
+function writeComparisons(comparisonResults: ComparisonResults): void {
+    const TEMPLATE = RESULT_PARSER_TO_COMPARE_TEMPLATE[config.resultParser];
+    const EXTENSION = RESULT_PARSER_TO_EXTENSION[config.resultParser];
+
+    // Directory should always be available but let's handle condition where
+    // user intentionally removes it during scan.
+    if (!fs.existsSync(RESULTS_COMPARE_LOCATION)) {
+        fs.mkdirSync(RESULTS_COMPARE_LOCATION, { recursive: true });
+    }
+
+    for (const type of ComparisonTypes) {
+        const results = comparisonResults[type];
+
+        if (results.length) {
+            const filename = `${type}${EXTENSION}`;
+            const content = TEMPLATE(type, results);
+
+            fs.writeFileSync(
+                `${RESULTS_COMPARE_LOCATION}/${filename}`,
+                content,
+                'utf8'
+            );
+        }
+    }
+}
+
+function equals(a: Result, b: Result): boolean {
+    return (
+        // Link contains path, extension, repository and repositoryOwner
+        a.link === b.link &&
+        a.rule === b.rule &&
+        a.message === b.message &&
+        a.source === b.source &&
+        a.error === b.error
+    );
+}

lib/file-client/result-templates.ts

@@ -16,6 +16,17 @@ export interface Result {
     error?: LintMessage['error'];
 }
 
+export interface ComparisonResults {
+    /** New results compared to previous scan */
+    added: Result[];
+
+    /** Removed results compared to previous scan */
+    removed: Result[];
+}
+
+type ComparisonType = keyof ComparisonResults;
+export const ComparisonTypes: ComparisonType[] = ['added', 'removed'];
+
 // prettier-ignore
 const RESULT_TEMPLATE_PLAINTEXT = (result: Result): string =>
 `Rule: ${result.rule}
@@ -50,6 +61,16 @@ ${result.error}
 export const RESULTS_TEMPLATE_CI_BASE = (result: Result): string =>
 `Repository: ${result.repositoryOwner}/${result.repository}`;
 
+// prettier-ignore
+const COMPARISON_RESULTS_TEMPLATE_PLAINTEXT = (type: ComparisonType, results: Result[]): string =>
+`${upperCaseFirstLetter(type)}:
+${results.map(RESULT_TEMPLATE_PLAINTEXT).join('\n')}`;
+
+// prettier-ignore
+const COMPARISON_RESULTS_TEMPLATE_MARKDOWN = (type: ComparisonType, results: Result[]): string =>
+`# ${upperCaseFirstLetter(type)}:
+${results.map(RESULT_TEMPLATE_MARKDOWN).join('\n')}`;
+
 export const RESULT_PARSER_TO_TEMPLATE: Record<
     ResultParser,
     (result: Result) => string
@@ -58,7 +79,21 @@ export const RESULT_PARSER_TO_TEMPLATE: Record<
     markdown: RESULT_TEMPLATE_MARKDOWN,
 } as const;
 
+export const RESULT_PARSER_TO_COMPARE_TEMPLATE: Record<
+    ResultParser,
+    (type: ComparisonType, results: Result[]) => string
+> = {
+    plaintext: COMPARISON_RESULTS_TEMPLATE_PLAINTEXT,
+    markdown: COMPARISON_RESULTS_TEMPLATE_MARKDOWN,
+} as const;
+
 export const RESULT_PARSER_TO_EXTENSION: Record<ResultParser, string> = {
     plaintext: '',
     markdown: '.md',
 } as const;
+
+function upperCaseFirstLetter(text: string) {
+    const [firstLetter, ...letters] = text;
+
+    return [firstLetter.toUpperCase(), ...letters].join('');
+}

lib/file-client/results-store.ts

@@ -1,10 +1,11 @@
-import { Result } from './result-templates';
+import { ComparisonResults, Result } from './result-templates';
 
 /**
  * A single source for the scan's results
  */
 class ResultsStore {
     private results: Result[] = [];
+    private comparisonResults: ComparisonResults | null = null;
 
     addResults(...results: Result[]) {
         this.results.push(...results);
@@ -13,6 +14,14 @@ class ResultsStore {
     getResults() {
         return this.results;
     }
+
+    setComparisonResults(comparisonResults: ComparisonResults) {
+        this.comparisonResults = comparisonResults;
+    }
+
+    getComparisonResults() {
+        return this.comparisonResults;
+    }
 }
 
 export default new ResultsStore();

lib/progress-logger/exit-handler.ts

@@ -1,14 +1,30 @@
 import config from '@config';
-import { ResultsStore } from '@file-client';
+import {
+    ResultsStore,
+    compareResults,
+    writeComparisonResults,
+} from '@file-client';
 
 /**
  * Callback invoked once scan is complete and application is about to exit
  */
 export default async function onExit(): Promise<void> {
+    const results = ResultsStore.getResults();
     const errors = [];
 
+    if (config.compare) {
+        try {
+            const comparisonResults = compareResults(results);
+            ResultsStore.setComparisonResults(comparisonResults);
+
+            writeComparisonResults(comparisonResults, results);
+        } catch (e) {
+            errors.push('Error occured while generating comparison results');
+            errors.push(e.stack);
+        }
+    }
+
     if (config.onComplete) {
-        const results = ResultsStore.getResults();
         try {
             const onCompletePromise = config.onComplete(results);
 

test/integration/integration.test.ts

@@ -1,11 +1,12 @@
 import fs from 'fs';
 
 import {
+    getResults,
+    getComparisonResults,
     runProductionBuild,
     INTEGRATION_REPO_OWNER,
     INTEGRATION_REPO_NAME,
     REPOSITORY_CACHE,
-    getResults,
 } from '../utils';
 
 describe('integration', () => {
@@ -586,4 +587,99 @@ describe('integration', () => {
         expect(exitCode).toBe(0);
         expect(output.pop()).toMatch(/Results:/);
     });
+
+    test('comparison results are generated', async () => {
+        await runProductionBuild({
+            compare: true,
+            CI: false,
+            rulesUnderTesting: [
+                'no-compare-neg-zero', // Used in initial scan, not in second
+                // 'no-undef', // Used in second scan, not in first
+                'no-empty', // Used in both scans
+            ],
+            eslintrc: {
+                root: true,
+                extends: ['eslint:all'],
+            },
+        });
+
+        await runProductionBuild({
+            compare: true,
+            CI: false,
+            rulesUnderTesting: [
+                // 'no-compare-neg-zero', // Used in initial scan, not in second
+                'no-undef', // Used in second scan, not in first
+                'no-empty', // Used in both scans
+            ],
+            eslintrc: {
+                root: true,
+                extends: ['eslint:all'],
+            },
+        });
+
+        // Remaining errors should be visible in results but not in comparison
+        expect(getResults()).toMatch(/no-empty/);
+
+        const comparisonResults = getComparisonResults();
+        const snapshot = [
+            '[ADDED]',
+            comparisonResults.added,
+            '[ADDED]',
+            '[REMOVED]',
+            comparisonResults.removed,
+            '[REMOVED]',
+        ].join('\n');
+
+        expect(snapshot).toMatchInlineSnapshot(`
+            "[ADDED]
+            # Added:
+            ## Rule: no-undef
+
+            -   Message: \`'window' is not defined.\`
+            -   Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/expected-to-crash-linter.js\`
+            -   [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/expected-to-crash-linter.js#L2-L2)
+
+            \`\`\`js
+            // Identifier.name = attributeForCrashing
+            window.attributeForCrashing();
+
+            \`\`\`
+
+            ## Rule: no-undef
+
+            -   Message: \`'bar' is not defined.\`
+            -   Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+            -   [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L1-L1)
+
+            \`\`\`js
+            var foo = bar;
+
+            if (foo) {
+            }
+
+            var p = {
+            \`\`\`
+
+            [ADDED]
+            [REMOVED]
+            # Removed:
+            ## Rule: no-compare-neg-zero
+
+            -   Message: \`Do not use the '===' operator to compare against -0.\`
+            -   Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+            -   [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L14-L14)
+
+            \`\`\`js
+            };
+            p.getName();
+
+
+            if (foo === -0) {
+              // prevent no-empty
+            }
+            \`\`\`
+
+            [REMOVED]"
+        `);
+    });
 });