From a9af053094545b1d1cc6bbdfcd71252d1247b5ae Mon Sep 17 00:00:00 2001 From: Christian Emmer <10749361+emmercm@users.noreply.github.com> Date: Sat, 22 Jun 2024 12:35:38 -0700 Subject: [PATCH] Feature: allow exact-matching of archives in DATs --- src/igir.ts | 27 +++++++++++++++-- src/modules/movedRomDeleter.ts | 52 +++++++++++++++++++++++---------- src/modules/romScanner.ts | 6 +++- src/modules/scanner.ts | 15 ++++++++-- src/types/files/archives/zip.ts | 2 +- test/fixtures/dats/one.dat | 3 +- test/igir.test.ts | 19 ++++++------ test/modules/romScanner.test.ts | 25 ++++++++++++---- 8 files changed, 111 insertions(+), 38 deletions(-) diff --git a/src/igir.ts b/src/igir.ts index b1226d73..9d2ec7e5 100644 --- a/src/igir.ts +++ b/src/igir.ts @@ -42,6 +42,7 @@ import DATStatus from './types/datStatus.js'; import File from './types/files/file.js'; import FileCache from './types/files/fileCache.js'; import { ChecksumBitmask } from './types/files/fileChecksums.js'; +import FileFactory from './types/files/fileFactory.js'; import IndexedFiles from './types/indexedFiles.js'; import Options from './types/options.js'; import OutputFactory from './types/outputFactory.js'; @@ -97,7 +98,10 @@ export default class Igir { // Scan and process input files let dats = await this.processDATScanner(); - const indexedRoms = await this.processROMScanner(this.determineScanningBitmask(dats)); + const indexedRoms = await this.processROMScanner( + this.determineScanningBitmask(dats), + this.determineScanningChecksumArchives(dats), + ); const roms = indexedRoms.getFiles(); const patches = await this.processPatchScanner(); @@ -309,11 +313,28 @@ export default class Igir { return matchChecksum; } - private async processROMScanner(checksumBitmask: number): Promise { + private determineScanningChecksumArchives(dats: DAT[]): boolean { + return dats + .some((dat) => dat.getGames() + .some((game) => game.getRoms() + .some((rom) => { + const isArchive = FileFactory.isExtensionArchive(rom.getName()); + if (isArchive) { + this.logger.trace(`${dat.getNameShort()}: contains archives, enabling checksum calculation of raw archive contents`); + } + return isArchive; + }))); + } + + private async processROMScanner( + checksumBitmask: number, + checksumArchives: boolean, + ): Promise { const romScannerProgressBarName = 'Scanning for ROMs'; const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName); - const rawRomFiles = await new ROMScanner(this.options, romProgressBar).scan(checksumBitmask); + const rawRomFiles = await new ROMScanner(this.options, romProgressBar) + .scan(checksumBitmask, checksumArchives); await romProgressBar.setName('Detecting ROM headers'); const romFilesWithHeaders = await new ROMHeaderProcessor(this.options, romProgressBar) diff --git a/src/modules/movedRomDeleter.ts b/src/modules/movedRomDeleter.ts index bcc06b9d..27fc5fe8 100644 --- a/src/modules/movedRomDeleter.ts +++ b/src/modules/movedRomDeleter.ts @@ -2,7 +2,9 @@ import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js'; import ArrayPoly from '../polyfill/arrayPoly.js'; import fsPoly from '../polyfill/fsPoly.js'; import DAT from '../types/dats/dat.js'; +import Archive from '../types/files/archives/archive.js'; import ArchiveEntry from '../types/files/archives/archiveEntry.js'; +import ArchiveFile from '../types/files/archives/archiveFile.js'; import File from '../types/files/file.js'; import Module from './module.js'; @@ -79,21 +81,41 @@ export default class MovedROMDeleter extends Module { movedEntries.flatMap((file) => file.hashCode()), ); - const inputEntries = groupedInputRoms.get(filePath) ?? []; - - const unmovedEntries = inputEntries.filter((entry) => { - if (entry instanceof ArchiveEntry - && movedEntries.length === 1 - && !(movedEntries[0] instanceof ArchiveEntry) - && movedEntries[0].getFilePath() === entry.getFilePath() - ) { - // If the input archive entry was written as a raw archive, then consider it moved - return false; - } - - // Otherwise, the entry needs to have been explicitly moved - return !movedEntryHashCodes.has(entry.hashCode()); - }); + const inputFilesForPath = groupedInputRoms.get(filePath) ?? []; + const inputFileIsArchive = inputFilesForPath + .some((inputFile) => inputFile instanceof ArchiveEntry); + + const unmovedFiles = inputFilesForPath + .filter((inputFile) => !(inputFile instanceof ArchiveEntry)) + // The input archive entry needs to have been explicitly moved + .filter((inputFile) => !movedEntryHashCodes.has(inputFile.hashCode())); + + if (inputFileIsArchive && unmovedFiles.length === 0) { + // The input file is an archive, and it was fully extracted OR the archive file itself was + // an exact match and was moved as-is + return filePath; + } + + const unmovedArchiveEntries = inputFilesForPath + .filter(( + inputFile, + ): inputFile is ArchiveEntry => inputFile instanceof ArchiveEntry) + .filter((inputEntry) => { + if (movedEntries.length === 1 && movedEntries[0] instanceof ArchiveFile) { + // If the input archive was written as a raw archive, then consider it moved + return false; + } + + // Otherwise, the input archive entry needs to have been explicitly moved + return !movedEntryHashCodes.has(inputEntry.hashCode()); + }); + + if (inputFileIsArchive && unmovedArchiveEntries.length === 0) { + // The input file is an archive and it was fully zipped + return filePath; + } + + const unmovedEntries = [...unmovedFiles, ...unmovedArchiveEntries]; if (unmovedEntries.length > 0) { this.progressBar.logWarn(`${filePath}: not deleting moved file, ${unmovedEntries.length.toLocaleString()} archive entr${unmovedEntries.length !== 1 ? 'ies were' : 'y was'} unmatched:\n${unmovedEntries.sort().map((entry) => ` ${entry}`).join('\n')}`); return undefined; diff --git a/src/modules/romScanner.ts b/src/modules/romScanner.ts index 185d5ccc..f84f9f46 100644 --- a/src/modules/romScanner.ts +++ b/src/modules/romScanner.ts @@ -16,7 +16,10 @@ export default class ROMScanner extends Scanner { /** * Scan for ROM files. */ - async scan(checksumBitmask: number = ChecksumBitmask.CRC32): Promise { + async scan( + checksumBitmask: number = ChecksumBitmask.CRC32, + checksumArchives = false, + ): Promise { this.progressBar.logTrace('scanning ROM files'); await this.progressBar.setSymbol(ProgressBarSymbol.SEARCHING); await this.progressBar.reset(0); @@ -31,6 +34,7 @@ export default class ROMScanner extends Scanner { romFilePaths, this.options.getReaderThreads(), checksumBitmask, + checksumArchives, ); this.progressBar.logTrace('done scanning ROM files'); diff --git a/src/modules/scanner.ts b/src/modules/scanner.ts index 14331f72..194fda5b 100644 --- a/src/modules/scanner.ts +++ b/src/modules/scanner.ts @@ -4,6 +4,7 @@ import DriveSemaphore from '../driveSemaphore.js'; import ElasticSemaphore from '../elasticSemaphore.js'; import ArrayPoly from '../polyfill/arrayPoly.js'; import fsPoly from '../polyfill/fsPoly.js'; +import ArchiveEntry from '../types/files/archives/archiveEntry.js'; import File from '../types/files/file.js'; import FileFactory from '../types/files/fileFactory.js'; import Options from '../types/options.js'; @@ -30,6 +31,7 @@ export default abstract class Scanner extends Module { filePaths: string[], threads: number, checksumBitmask: number, + checksumArchives = false, ): Promise { return (await new DriveSemaphore(threads).map( filePaths, @@ -38,7 +40,7 @@ export default abstract class Scanner extends Module { const waitingMessage = `${inputFile} ...`; this.progressBar.addWaitingMessage(waitingMessage); - const files = await this.getFilesFromPath(inputFile, checksumBitmask); + const files = await this.getFilesFromPath(inputFile, checksumBitmask, checksumArchives); this.progressBar.removeWaitingMessage(waitingMessage); await this.progressBar.incrementDone(); @@ -60,6 +62,7 @@ export default abstract class Scanner extends Module { private async getFilesFromPath( filePath: string, checksumBitmask: number, + checksumArchives = false, ): Promise { try { const totalKilobytes = await fsPoly.size(filePath) / 1024; @@ -72,7 +75,15 @@ export default abstract class Scanner extends Module { return []; } } - return FileFactory.filesFrom(filePath, checksumBitmask); + + const filesFromPath = await FileFactory.filesFrom(filePath, checksumBitmask); + + const fileIsArchive = filesFromPath.some((file) => file instanceof ArchiveEntry); + if (checksumArchives && fileIsArchive) { + filesFromPath.push(await FileFactory.fileFrom(filePath, checksumBitmask)); + } + + return filesFromPath; }, totalKilobytes, ); diff --git a/src/types/files/archives/zip.ts b/src/types/files/archives/zip.ts index 8ab0e23a..485e2dce 100644 --- a/src/types/files/archives/zip.ts +++ b/src/types/files/archives/zip.ts @@ -22,7 +22,7 @@ export default class Zip extends Archive { } static getExtensions(): string[] { - return ['.zip']; + return ['.zip', '.apk', '.ipa', '.jar', '.pk3']; } // eslint-disable-next-line class-methods-use-this diff --git a/test/fixtures/dats/one.dat b/test/fixtures/dats/one.dat index 0fb42a40..beb7461a 100644 --- a/test/fixtures/dats/one.dat +++ b/test/fixtures/dats/one.dat @@ -29,7 +29,8 @@ Lorem Ipsum - + + One Three diff --git a/test/igir.test.ts b/test/igir.test.ts index 06cf89fd..e7545384 100644 --- a/test/igir.test.ts +++ b/test/igir.test.ts @@ -172,7 +172,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'], [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -216,7 +216,7 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ // Fizzbuzz.nes is explicitly missing! ['Foobar.lnx', 'b22c9747'], - ['Lorem Ipsum.rom', '70856527'], + ['Lorem Ipsum.zip|loremipsum.rom', '70856527'], [`${path.join('One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -272,7 +272,6 @@ describe('with explicit DATs', () => { [path.join('nes', 'smdb', 'Hardware Target Game Database', 'Dummy', 'Fizzbuzz.nes'), '370517b5'], ['one.rom', '00000000'], // explicitly not deleted, it is not in an extension subdirectory [`${path.join('rar', 'Headered', 'LCDTestROM.lnx.rar')}|LCDTestROM.lnx`, '2d251538'], - [path.join('rom', 'One', 'Lorem Ipsum.rom'), '70856527'], [path.join('rom', 'One', 'Three Four Five', 'Five.rom'), '3e5daf67'], [path.join('rom', 'One', 'Three Four Five', 'Four.rom'), '1cf3ca74'], [path.join('rom', 'One', 'Three Four Five', 'Three.rom'), 'ff46c5d8'], @@ -290,6 +289,7 @@ describe('with explicit DATs', () => { [path.join('rom', 'smdb', 'Hardware Target Game Database', 'Patchable', 'C01173E.rom'), 'dfaebe28'], [path.join('smc', 'Headered', 'speed_test_v51.smc'), '9adca6cc'], [`${path.join('zip', 'Headered', 'fds_joypad_test.fds.zip')}|fds_joypad_test.fds`, '1e58456d'], + [`${path.join('zip', 'One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -326,7 +326,6 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], @@ -365,7 +364,6 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], @@ -419,7 +417,7 @@ describe('with explicit DATs', () => { [path.join('igir combined', 'KDULVQN.rom'), 'b1c303e4'], [path.join('igir combined', 'LCDTestROM.lnx'), '2d251538'], [path.join('igir combined', 'LCDTestROM.lyx'), '42583855'], - [path.join('igir combined', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('igir combined', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [path.join('igir combined', 'One Three', 'One.rom'), 'f817a89f'], [path.join('igir combined', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('igir combined', 'speed_test_v51.sfc'), '8beffd94'], @@ -453,6 +451,7 @@ describe('with explicit DATs', () => { path.join('raw', 'loremipsum.rom'), path.join('raw', 'one.rom'), path.join('raw', 'three.rom'), + path.join('zip', 'loremipsum.zip'), ]); expect(result.cleanedFiles).toHaveLength(0); }); @@ -554,7 +553,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.zip')}|speed_test_v51.sfc`, '8beffd94'], [`${path.join('One', 'Fizzbuzz.zip')}|Fizzbuzz.nes`, '370517b5'], [`${path.join('One', 'Foobar.zip')}|Foobar.lnx`, 'b22c9747'], - [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.rom`, '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.zip`, '7ee77289'], [`${path.join('One', 'One Three.zip')}|One.rom`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|Three.rom`, 'ff46c5d8'], [`${path.join('One', 'Three Four Five.zip')}|Five.rom`, '3e5daf67'], @@ -613,7 +612,7 @@ describe('with explicit DATs', () => { ['Headerless.zip|speed_test_v51.sfc', '8beffd94'], ['One.zip|Fizzbuzz.nes', '370517b5'], ['One.zip|Foobar.lnx', 'b22c9747'], - ['One.zip|Lorem Ipsum.rom', '70856527'], + ['One.zip|Lorem Ipsum.zip', '7ee77289'], [`One.zip|${path.join('One Three', 'One.rom')}`, 'f817a89f'], [`One.zip|${path.join('One Three', 'Three.rom')}`, 'ff46c5d8'], [`One.zip|${path.join('Three Four Five', 'Five.rom')}`, '3e5daf67'], @@ -658,7 +657,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc -> ${path.join('', 'headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'], [`${path.join('One', 'Fizzbuzz.nes')} -> ${path.join('', 'raw', 'fizzbuzz.nes')}`, '370517b5'], [`${path.join('One', 'Foobar.lnx')} -> ${path.join('', 'foobar.lnx')}`, 'b22c9747'], - [`${path.join('One', 'Lorem Ipsum.rom')} -> ${path.join('', 'raw', 'loremipsum.rom')}`, '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom -> ${path.join('', 'zip', 'loremipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -715,7 +714,7 @@ describe('with explicit DATs', () => { [path.join('Headerless', 'speed_test_v51.sfc'), '8beffd94'], [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], diff --git a/test/modules/romScanner.test.ts b/test/modules/romScanner.test.ts index 677f6f7c..7eb7d9cc 100644 --- a/test/modules/romScanner.test.ts +++ b/test/modules/romScanner.test.ts @@ -4,7 +4,8 @@ import path from 'node:path'; import Constants from '../../src/constants.js'; import ROMScanner from '../../src/modules/romScanner.js'; import fsPoly from '../../src/polyfill/fsPoly.js'; -import Options from '../../src/types/options.js'; +import { ChecksumBitmask } from '../../src/types/files/fileChecksums.js'; +import Options, { OptionsProps } from '../../src/types/options.js'; import ProgressBarFake from '../console/progressBarFake.js'; function createRomScanner(input: string[], inputExclude: string[] = []): ROMScanner { @@ -40,6 +41,18 @@ describe('multiple files', () => { await expect(createRomScanner(['test/fixtures/roms/**/*', 'test/fixtures/roms/**/*.{rom,zip}']).scan()).resolves.toHaveLength(expectedRomFiles); }); + test.each([ + [{ input: ['test/fixtures/roms'] }, 90], + [{ input: ['test/fixtures/roms/7z'] }, 12], + [{ input: ['test/fixtures/roms/rar'] }, 12], + [{ input: ['test/fixtures/roms/tar'] }, 12], + [{ input: ['test/fixtures/roms/zip'] }, 15], + ] satisfies [OptionsProps, number][])('should calculate checksums of archives: %s', async (optionsProps, expectedRomFiles) => { + const scannedFiles = await new ROMScanner(new Options(optionsProps), new ProgressBarFake()) + .scan(ChecksumBitmask.CRC32, true); + expect(scannedFiles).toHaveLength(expectedRomFiles); + }); + it('should scan multiple files with some file exclusions', async () => { await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44); await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom', 'test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44); @@ -169,8 +182,10 @@ describe('multiple files', () => { }); }); -it('should scan single files', async () => { - await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1); - await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1); - await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1); +describe('single files', () => { + it('should scan single files with no exclusions', async () => { + await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1); + await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1); + await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1); + }); });