fix: cspell-tools: be able to update shasum checksum files.

streetsidesoftware · Jul 14, 2023 · e53d4a4 · e53d4a4
1 parent 558124c
commit e53d4a4
Show file tree

Hide file tree

Showing 7 changed files with 202 additions and 30 deletions.
diff --git a/packages/cspell-tools/bin.js b/packages/cspell-tools/bin.js
@@ -6,6 +6,8 @@ import { run } from './dist/app.js';
 run(program, process.argv).catch((e) => {
     if (!(e instanceof CommanderError)) {
         console.log(e);
+    } else {
+        console.log(e.message);
     }
     process.exitCode = 1;
 });
diff --git a/packages/cspell-tools/fixtures/dicts/source-files.txt b/packages/cspell-tools/fixtures/dicts/source-files.txt
@@ -0,0 +1,3 @@
+
+colors.txt
+cities.txt
diff --git a/packages/cspell-tools/src/app.ts b/packages/cspell-tools/src/app.ts
@@ -1,7 +1,7 @@
 // For large dictionaries, it is necessary to increase the memory limit.
 
 import type * as program from 'commander';
-import { CommanderError } from 'commander';
+import { CommanderError, Option } from 'commander';
 import { readFileSync } from 'fs';
 
 import type { CompileAppOptions, CompileTrieAppOptions } from './AppOptions.js';
@@ -11,7 +11,7 @@ import * as compiler from './compiler/index.js';
 import { logWithTimestamp } from './compiler/logWithTimestamp.js';
 import type { FeatureFlags } from './FeatureFlags/index.js';
 import { gzip } from './gzip/index.js';
-import { reportCheckChecksumFile, reportChecksumForFiles } from './shasum/shasum.js';
+import { reportCheckChecksumFile, reportChecksumForFiles, updateChecksumForFiles } from './shasum/shasum.js';
 import { toError } from './util/errors.js';
 
 const npmPackageRaw = readFileSync(new URL('../package.json', import.meta.url), 'utf8');
@@ -57,7 +57,9 @@ function addCompileOptions(compileCommand: program.Command): program.Command {
 
 interface ShasumOptions {
     check?: string | undefined;
+    update?: string | undefined;
     root?: string | undefined;
+    listFile?: string[] | undefined;
 }
 
 export async function run(program: program.Command, argv: string[], flags?: FeatureFlags): Promise<void> {
@@ -72,8 +74,10 @@ export async function run(program: program.Command, argv: string[], flags?: Feat
 
     async function shasum(files: string[], options: ShasumOptions): Promise<void> {
         const report = options.check
-            ? await reportCheckChecksumFile(options.check, files, options.root)
-            : await reportChecksumForFiles(files, options.root);
+            ? await reportCheckChecksumFile(options.check, files, options)
+            : options.update
+            ? await updateChecksumForFiles(options.update, files, options)
+            : await reportChecksumForFiles(files, options);
         console.log('%s', report.report);
 
         if (!report.passed) {
@@ -114,10 +118,14 @@ export async function run(program: program.Command, argv: string[], flags?: Feat
     program
         .command('shasum [files...]')
         .description('Calculate the checksum for files.')
+        .option('--list-file <list-file.txt...>', 'Specify one or more files that contain paths of files to check.')
         .option(
             '-c, --check <checksum.txt>',
             'Verify the checksum of files against those stored in the checksum.txt file.'
         )
+        .addOption(
+            new Option('-u, --update <checksum.txt>', 'Update checksums found in the file.').conflicts('--check')
+        )
         .option(
             '-r, --root <root>',
             'Specify the root to use for relative paths. The current working directory is used by default.'

diff --git a/packages/cspell-tools/src/shasum/__snapshots__/shasum.test.ts.snap b/packages/cspell-tools/src/shasum/__snapshots__/shasum.test.ts.snap
@@ -1,5 +1,27 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`shasum > calcUpdateChecksumForFiles '_checksum-failed.txt' 1`] = `
+"a8ab65e8305f4911577525c8e950fcea6667de59  cities.trie
+963a65138d4391c8de2f0dfb5a7ef890e512a95e  cities.trie.gz
+477e8bc9954033392e432fd66f0d4278884bde38  cities.txt
+55915445afc07bf877eea1e982aecb7c97b247da  colors.trie
+3a6b55a089d018878e8b904f8f19391f2e30b66c  colors.txt
+25a493fa62702d3e052717a26c6740a30614457e  sampleCodeDic.txt
+"
+`;
+
+exports[`shasum > calcUpdateChecksumForFiles '_checksum-missing-file' 1`] = `
+"477e8bc9954033392e432fd66f0d4278884bde38  cities.txt
+3a6b55a089d018878e8b904f8f19391f2e30b66c  colors.txt
+"
+`;
+
+exports[`shasum > calcUpdateChecksumForFiles 'new_checksum_file.txt' 1`] = `
+"477e8bc9954033392e432fd66f0d4278884bde38  cities.txt
+3a6b55a089d018878e8b904f8f19391f2e30b66c  colors.txt
+"
+`;
+
 exports[`shasum > checkShasumFile not pass 1`] = `
 [
   {
@@ -71,7 +93,17 @@ exports[`shasum > checkShasumFile pass with files 1`] = `
 ]
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum.txt' [ 'colors.txt', 'my_cities.txt' ] 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum.txt' [ 'colors.txt', 'my_cities.txt' ] 'source-files.txt' 1`] = `
+{
+  "passed": false,
+  "report": "colors.txt: OK
+my_cities.txt: FAILED - Missing Checksum.
+cities.txt: OK
+shasum: WARNING: 1 computed checksum did NOT match",
+}
+`;
+
+exports[`shasum > reportCheckChecksumFile '_checksum.txt' [ 'colors.txt', 'my_cities.txt' ] undefined 1`] = `
 {
   "passed": false,
   "report": "colors.txt: OK
@@ -80,7 +112,7 @@ shasum: WARNING: 1 computed checksum did NOT match",
 }
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum.txt' undefined 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum.txt' undefined undefined 1`] = `
 {
   "passed": true,
   "report": "cities.trie: OK
@@ -92,7 +124,7 @@ sampleCodeDic.txt: OK",
 }
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum-failed.txt' undefined 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum-failed.txt' undefined undefined 1`] = `
 {
   "passed": false,
   "report": "cities.trie: OK
@@ -105,15 +137,15 @@ shasum: WARNING: 1 computed checksum did NOT match",
 }
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum-failed2.txt' [ 'colors.txt', 'cities.txt' ] 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum-failed2.txt' [ 'cities.txt', 'colors.txt' ] undefined 1`] = `
 {
   "passed": true,
-  "report": "colors.txt: OK
-cities.txt: OK",
+  "report": "cities.txt: OK
+colors.txt: OK",
 }
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum-failed2.txt' undefined 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum-failed2.txt' undefined undefined 1`] = `
 {
   "passed": false,
   "report": "cities.trie: OK
@@ -127,7 +159,7 @@ shasum: WARNING: 2 computed checksums did NOT match",
 }
 `;
 
-exports[`shasum > reportCheckChecksumFile '_checksum-missing-file.txt' undefined 1`] = `
+exports[`shasum > reportCheckChecksumFile '_checksum-missing-file.txt' undefined undefined 1`] = `
 {
   "passed": false,
   "report": "missing-file.txt: FAILED - Failed to read file.

diff --git a/packages/cspell-tools/src/shasum/shasum.test.ts b/packages/cspell-tools/src/shasum/shasum.test.ts
@@ -1,9 +1,32 @@
-import { describe, expect, test } from 'vitest';
+import { writeFile } from 'node:fs/promises';
+
+import { afterEach, describe, expect, test, vi } from 'vitest';
 
 import { resolvePathToFixture } from '../test/TestHelper.js';
-import { checkShasumFile, reportCheckChecksumFile, reportChecksumForFiles } from './shasum.js';
+import {
+    calcUpdateChecksumForFiles,
+    checkShasumFile,
+    reportCheckChecksumFile,
+    reportChecksumForFiles,
+    updateChecksumForFiles,
+} from './shasum.js';
+
+vi.mock('node:fs/promises', async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const fs: any = await vi.importActual('node:fs/promises');
+    return {
+        ...fs,
+        writeFile: vi.fn().mockImplementation(() => Promise.resolve(undefined)),
+    };
+});
+
+const mockedWriteFile = vi.mocked(writeFile);
 
 describe('shasum', () => {
+    afterEach(() => {
+        vi.resetAllMocks();
+    });
+
     test('checkShasumFile pass', async () => {
         const root = resolvePathToFixture('dicts');
         const filename = resolvePathToFixture('dicts/_checksum.txt');
@@ -61,21 +84,48 @@ describe('shasum', () => {
     test('reportChecksumForFiles', async () => {
         const root = resolvePathToFixture('dicts');
         const files = ['colors.txt', 'cities.txt'];
-        const report = await reportChecksumForFiles(files, root);
+        const report = await reportChecksumForFiles(files, { root });
         expect(report).toMatchSnapshot();
     });
 
     test.each`
-        filename                        | files
-        ${'_checksum.txt'}              | ${undefined}
-        ${'_checksum.txt'}              | ${['colors.txt', 'my_cities.txt']}
-        ${'_checksum-failed.txt'}       | ${undefined}
-        ${'_checksum-failed2.txt'}      | ${undefined}
-        ${'_checksum-failed2.txt'}      | ${['colors.txt', 'cities.txt']}
-        ${'_checksum-missing-file.txt'} | ${undefined}
-    `('reportCheckChecksumFile $filename $files', async ({ filename, files }) => {
+        filename                        | files                              | listFile
+        ${'_checksum.txt'}              | ${undefined}                       | ${undefined}
+        ${'_checksum.txt'}              | ${['colors.txt', 'my_cities.txt']} | ${undefined}
+        ${'_checksum-failed.txt'}       | ${undefined}                       | ${undefined}
+        ${'_checksum-failed2.txt'}      | ${undefined}                       | ${undefined}
+        ${'_checksum-failed2.txt'}      | ${['cities.txt', 'colors.txt']}    | ${undefined}
+        ${'_checksum-missing-file.txt'} | ${undefined}                       | ${undefined}
+        ${'_checksum.txt'}              | ${['colors.txt', 'my_cities.txt']} | ${'source-files.txt'}
+    `('reportCheckChecksumFile $filename $files $listFile', async ({ filename, files, listFile }) => {
         const root = resolvePathToFixture('dicts');
-        const report = await reportCheckChecksumFile(resolvePathToFixture('dicts', filename), files, root);
+        const report = await reportCheckChecksumFile(resolvePathToFixture('dicts', filename), files, {
+            root,
+            listFile: listFile ? [resolvePathToFixture('dicts', listFile)] : undefined,
+        });
         expect(report).toMatchSnapshot();
     });
+
+    test.each`
+        filename
+        ${'_checksum-failed.txt'}
+        ${'_checksum-missing-file'}
+        ${'new_checksum_file.txt'}
+    `('calcUpdateChecksumForFiles $filename', async ({ filename }) => {
+        const root = resolvePathToFixture('dicts');
+        const checksumFile = resolvePathToFixture('dicts', filename);
+        const listFile = resolvePathToFixture('dicts', 'source-files.txt');
+
+        const result = await calcUpdateChecksumForFiles(checksumFile, [], { root, listFile: [listFile] });
+        expect(result).toMatchSnapshot();
+    });
+
+    test('updateChecksumForFiles', async () => {
+        const checksumFile = 'temp/my-checksum.txt';
+        const root = resolvePathToFixture('dicts');
+        const listFile = resolvePathToFixture('dicts', 'source-files.txt');
+
+        const result = await updateChecksumForFiles(checksumFile, [], { root, listFile: [listFile] });
+        expect(mockedWriteFile).toHaveBeenLastCalledWith(checksumFile, result.report);
+    });
 });
diff --git a/packages/cspell-tools/src/shasum/shasum.ts b/packages/cspell-tools/src/shasum/shasum.ts
@@ -1,6 +1,7 @@
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
+import { toError } from '../util/errors.js';
 import { isDefined } from '../util/index.js';
 import { calcFileChecksum, checkFile } from './checksum.js';
 
@@ -104,10 +105,17 @@ interface ReportResult {
     passed: boolean;
 }
 
-export async function reportChecksumForFiles(files: string[], root: string | undefined): Promise<ReportResult> {
+interface ReportOptions {
+    root?: string | undefined;
+    listFile?: string[];
+}
+
+export async function reportChecksumForFiles(files: string[], options: ReportOptions): Promise<ReportResult> {
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
     let numFailed = 0;
     const result = await Promise.all(
-        files.map((file) =>
+        filesToCheck.map((file) =>
             shasumFile(file, root).catch((e) => {
                 ++numFailed;
                 if (typeof e !== 'string') throw e;
@@ -123,9 +131,11 @@ export async function reportChecksumForFiles(files: string[], root: string | und
 export async function reportCheckChecksumFile(
     filename: string,
     files: string[] | undefined,
-    root: string | undefined
+    options: ReportOptions
 ): Promise<ReportResult> {
-    const result = await checkShasumFile(filename, files, root);
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const result = await checkShasumFile(filename, filesToCheck, root);
     const lines = result.map(({ filename, passed, error }) =>
         `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim()
     );
@@ -138,3 +148,66 @@ export async function reportCheckChecksumFile(
     }
     return { report: lines.join('\n'), passed };
 }
+
+async function resolveFileList(files: string[] | undefined, listFile: string[] | undefined): Promise<string[]> {
+    files = files || [];
+    listFile = listFile || [];
+
+    const setOfFiles = new Set(files);
+
+    const pending = listFile.map((filename) => readFile(filename, 'utf8'));
+
+    for await (const content of pending) {
+        content
+            .split('\n')
+            .map((a) => a.trim())
+            .filter((a) => a)
+            .forEach((file) => setOfFiles.add(file));
+    }
+    return [...setOfFiles];
+}
+
+export async function calcUpdateChecksumForFiles(
+    filename: string,
+    files: string[],
+    options: ReportOptions
+): Promise<string> {
+    const root = options.root || '.';
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const currentEntries = await readAndParseShasumFile(filename).catch((err) => {
+        const e = toError(err);
+        if (e.code !== 'ENOENT') throw e;
+        return [] as ChecksumEntry[];
+    });
+    const entriesToUpdate = new Set([...filesToCheck, ...currentEntries.map((e) => e.filename)]);
+    const mustExist = new Set(filesToCheck);
+
+    const checksumMap = new Map(currentEntries.map(({ filename, checksum }) => [filename, checksum]));
+
+    for (const file of entriesToUpdate) {
+        try {
+            const checksum = await calcFileChecksum(resolve(root, file));
+            checksumMap.set(file, checksum);
+        } catch (e) {
+            if (mustExist.has(file) || toError(e).code !== 'ENOENT') throw e;
+            checksumMap.delete(file);
+        }
+    }
+
+    const updatedEntries = [...checksumMap]
+        .map(([filename, checksum]) => ({ filename, checksum }))
+        .sort((a, b) => (a.filename < b.filename ? -1 : 1));
+    return updatedEntries.map((e) => `${e.checksum}  ${e.filename}`).join('\n') + '\n';
+}
+
+export async function updateChecksumForFiles(
+    filename: string,
+    files: string[],
+    options: ReportOptions
+): Promise<ReportResult> {
+    const content = await calcUpdateChecksumForFiles(filename, files, options);
+
+    await writeFile(filename, content);
+
+    return { passed: true, report: content };
+}
diff --git a/packages/cspell-tools/src/util/errors.ts b/packages/cspell-tools/src/util/errors.ts
@@ -1,4 +1,8 @@
-export function toError(err: unknown): Error {
+export interface NodeError extends Error {
+    code?: string;
+}
+
+export function toError(err: unknown): NodeError {
     if (isError(err)) return err;
     return new Error(`${err}`);
 }