Feature: allow exact-matching of archives in DATs

src/igir.ts

@@ -42,6 +42,7 @@ import DATStatus from './types/datStatus.js';
 import File from './types/files/file.js';
 import FileCache from './types/files/fileCache.js';
 import { ChecksumBitmask } from './types/files/fileChecksums.js';
+import FileFactory from './types/files/fileFactory.js';
 import IndexedFiles from './types/indexedFiles.js';
 import Options from './types/options.js';
 import OutputFactory from './types/outputFactory.js';
@@ -97,7 +98,10 @@ export default class Igir {
 
     // Scan and process input files
     let dats = await this.processDATScanner();
-    const indexedRoms = await this.processROMScanner(this.determineScanningBitmask(dats));
+    const indexedRoms = await this.processROMScanner(
+      this.determineScanningBitmask(dats),
+      this.determineScanningChecksumArchives(dats),
+    );
     const roms = indexedRoms.getFiles();
     const patches = await this.processPatchScanner();
 
@@ -309,11 +313,28 @@ export default class Igir {
     return matchChecksum;
   }
 
-  private async processROMScanner(checksumBitmask: number): Promise<IndexedFiles> {
+  private determineScanningChecksumArchives(dats: DAT[]): boolean {
+    return dats
+      .some((dat) => dat.getGames()
+        .some((game) => game.getRoms()
+          .some((rom) => {
+            const isArchive = FileFactory.isExtensionArchive(rom.getName());
+            if (isArchive) {
+              this.logger.trace(`${dat.getNameShort()}: contains archives, enabling checksum calculation of raw archive contents`);
+            }
+            return isArchive;
+          })));
+  }
+
+  private async processROMScanner(
+    checksumBitmask: number,
+    checksumArchives: boolean,
+  ): Promise<IndexedFiles> {
     const romScannerProgressBarName = 'Scanning for ROMs';
     const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName);
 
-    const rawRomFiles = await new ROMScanner(this.options, romProgressBar).scan(checksumBitmask);
+    const rawRomFiles = await new ROMScanner(this.options, romProgressBar)
+      .scan(checksumBitmask, checksumArchives);
 
     await romProgressBar.setName('Detecting ROM headers');
     const romFilesWithHeaders = await new ROMHeaderProcessor(this.options, romProgressBar)

src/modules/movedRomDeleter.ts

@@ -2,7 +2,9 @@ import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
 import ArrayPoly from '../polyfill/arrayPoly.js';
 import fsPoly from '../polyfill/fsPoly.js';
 import DAT from '../types/dats/dat.js';
+import Archive from '../types/files/archives/archive.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
+import ArchiveFile from '../types/files/archives/archiveFile.js';
 import File from '../types/files/file.js';
 import Module from './module.js';
 
@@ -79,21 +81,41 @@ export default class MovedROMDeleter extends Module {
           movedEntries.flatMap((file) => file.hashCode()),
         );
 
-        const inputEntries = groupedInputRoms.get(filePath) ?? [];
-
-        const unmovedEntries = inputEntries.filter((entry) => {
-          if (entry instanceof ArchiveEntry
-            && movedEntries.length === 1
-            && !(movedEntries[0] instanceof ArchiveEntry)
-            && movedEntries[0].getFilePath() === entry.getFilePath()
-          ) {
-            // If the input archive entry was written as a raw archive, then consider it moved
-            return false;
-          }
-
-          // Otherwise, the entry needs to have been explicitly moved
-          return !movedEntryHashCodes.has(entry.hashCode());
-        });
+        const inputFilesForPath = groupedInputRoms.get(filePath) ?? [];
+        const inputFileIsArchive = inputFilesForPath
+          .some((inputFile) => inputFile instanceof ArchiveEntry);
+
+        const unmovedFiles = inputFilesForPath
+          .filter((inputFile) => !(inputFile instanceof ArchiveEntry))
+          // The input archive entry needs to have been explicitly moved
+          .filter((inputFile) => !movedEntryHashCodes.has(inputFile.hashCode()));
+
+        if (inputFileIsArchive && unmovedFiles.length === 0) {
+          // The input file is an archive, and it was fully extracted OR the archive file itself was
+          // an exact match and was moved as-is
+          return filePath;
+        }
+
+        const unmovedArchiveEntries = inputFilesForPath
+          .filter((
+            inputFile,
+          ): inputFile is ArchiveEntry<Archive> => inputFile instanceof ArchiveEntry)
+          .filter((inputEntry) => {
+            if (movedEntries.length === 1 && movedEntries[0] instanceof ArchiveFile) {
+              // If the input archive was written as a raw archive, then consider it moved
+              return false;
+            }
+
+            // Otherwise, the input archive entry needs to have been explicitly moved
+            return !movedEntryHashCodes.has(inputEntry.hashCode());
+          });
+
+        if (inputFileIsArchive && unmovedArchiveEntries.length === 0) {
+          // The input file is an archive and it was fully zipped
+          return filePath;
+        }
+
+        const unmovedEntries = [...unmovedFiles, ...unmovedArchiveEntries];
         if (unmovedEntries.length > 0) {
           this.progressBar.logWarn(`${filePath}: not deleting moved file, ${unmovedEntries.length.toLocaleString()} archive entr${unmovedEntries.length !== 1 ? 'ies were' : 'y was'} unmatched:\n${unmovedEntries.sort().map((entry) => `  ${entry}`).join('\n')}`);
           return undefined;

src/modules/romScanner.ts

@@ -16,7 +16,10 @@ export default class ROMScanner extends Scanner {
   /**
    * Scan for ROM files.
    */
-  async scan(checksumBitmask: number = ChecksumBitmask.CRC32): Promise<File[]> {
+  async scan(
+    checksumBitmask: number = ChecksumBitmask.CRC32,
+    checksumArchives = false,
+  ): Promise<File[]> {
     this.progressBar.logTrace('scanning ROM files');
     await this.progressBar.setSymbol(ProgressBarSymbol.SEARCHING);
     await this.progressBar.reset(0);
@@ -31,6 +34,7 @@ export default class ROMScanner extends Scanner {
       romFilePaths,
       this.options.getReaderThreads(),
       checksumBitmask,
+      checksumArchives,
     );
 
     this.progressBar.logTrace('done scanning ROM files');

src/modules/scanner.ts

@@ -4,6 +4,7 @@ import DriveSemaphore from '../driveSemaphore.js';
 import ElasticSemaphore from '../elasticSemaphore.js';
 import ArrayPoly from '../polyfill/arrayPoly.js';
 import fsPoly from '../polyfill/fsPoly.js';
+import ArchiveEntry from '../types/files/archives/archiveEntry.js';
 import File from '../types/files/file.js';
 import FileFactory from '../types/files/fileFactory.js';
 import Options from '../types/options.js';
@@ -30,6 +31,7 @@ export default abstract class Scanner extends Module {
     filePaths: string[],
     threads: number,
     checksumBitmask: number,
+    checksumArchives = false,
   ): Promise<File[]> {
     return (await new DriveSemaphore(threads).map(
       filePaths,
@@ -38,7 +40,7 @@ export default abstract class Scanner extends Module {
         const waitingMessage = `${inputFile} ...`;
         this.progressBar.addWaitingMessage(waitingMessage);
 
-        const files = await this.getFilesFromPath(inputFile, checksumBitmask);
+        const files = await this.getFilesFromPath(inputFile, checksumBitmask, checksumArchives);
 
         this.progressBar.removeWaitingMessage(waitingMessage);
         await this.progressBar.incrementDone();
@@ -60,6 +62,7 @@ export default abstract class Scanner extends Module {
   private async getFilesFromPath(
     filePath: string,
     checksumBitmask: number,
+    checksumArchives = false,
   ): Promise<File[]> {
     try {
       const totalKilobytes = await fsPoly.size(filePath) / 1024;
@@ -72,7 +75,15 @@ export default abstract class Scanner extends Module {
               return [];
             }
           }
-          return FileFactory.filesFrom(filePath, checksumBitmask);
+
+          const filesFromPath = await FileFactory.filesFrom(filePath, checksumBitmask);
+
+          const fileIsArchive = filesFromPath.some((file) => file instanceof ArchiveEntry);
+          if (checksumArchives && fileIsArchive) {
+            filesFromPath.push(await FileFactory.fileFrom(filePath, checksumBitmask));
+          }
+
+          return filesFromPath;
         },
         totalKilobytes,
       );

src/types/files/archives/zip.ts

@@ -22,7 +22,7 @@ export default class Zip extends Archive {
   }
 
   static getExtensions(): string[] {
-    return ['.zip'];
+    return ['.zip', '.apk', '.ipa', '.jar', '.pk3'];
   }
 
   // eslint-disable-next-line class-methods-use-this

test/fixtures/dats/one.dat

@@ -29,7 +29,8 @@
   <machine name="Lorem Ipsum">
     <description>Lorem Ipsum</description>
     <release name="Lorem Ipsum" region="JPN"/>
-    <rom name="Lorem Ipsum.rom" size="11" crc="70856527" md5="fffcb698d88fbc9425a636ba7e4712a3" sha1="1d913738eb363a4056c19e158aa81189a1eb7a55" status="verified"/>
+    <!--<rom name="Lorem Ipsum.rom" size="11" crc="70856527" md5="fffcb698d88fbc9425a636ba7e4712a3" sha1="1d913738eb363a4056c19e158aa81189a1eb7a55" status="verified"/>-->
+    <rom name="Lorem Ipsum.zip" size="203" crc="7ee77289" md5="9d4f876e42a8da0d4ae6f24c665476d9" sha1="25265aea64c1a5809d1b06cb5294a8293fb7027a" status="verified"/>
   </machine>
   <machine name="One Three">
     <description>One Three</description>

test/igir.test.ts

@@ -172,7 +172,7 @@ describe('with explicit DATs', () => {
         [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'],
         [path.join('One', 'Fizzbuzz.nes'), '370517b5'],
         [path.join('One', 'Foobar.lnx'), 'b22c9747'],
-        [path.join('One', 'Lorem Ipsum.rom'), '70856527'],
+        [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'],
         [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'],
         [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'],
         [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'],
@@ -216,7 +216,7 @@ describe('with explicit DATs', () => {
       expect(result.outputFilesAndCrcs).toEqual([
         // Fizzbuzz.nes is explicitly missing!
         ['Foobar.lnx', 'b22c9747'],
-        ['Lorem Ipsum.rom', '70856527'],
+        ['Lorem Ipsum.zip|loremipsum.rom', '70856527'],
         [`${path.join('One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'],
         [`${path.join('One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'],
         [`${path.join('One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'],
@@ -272,7 +272,6 @@ describe('with explicit DATs', () => {
         [path.join('nes', 'smdb', 'Hardware Target Game Database', 'Dummy', 'Fizzbuzz.nes'), '370517b5'],
         ['one.rom', '00000000'], // explicitly not deleted, it is not in an extension subdirectory
         [`${path.join('rar', 'Headered', 'LCDTestROM.lnx.rar')}|LCDTestROM.lnx`, '2d251538'],
-        [path.join('rom', 'One', 'Lorem Ipsum.rom'), '70856527'],
         [path.join('rom', 'One', 'Three Four Five', 'Five.rom'), '3e5daf67'],
         [path.join('rom', 'One', 'Three Four Five', 'Four.rom'), '1cf3ca74'],
         [path.join('rom', 'One', 'Three Four Five', 'Three.rom'), 'ff46c5d8'],
@@ -290,6 +289,7 @@ describe('with explicit DATs', () => {
         [path.join('rom', 'smdb', 'Hardware Target Game Database', 'Patchable', 'C01173E.rom'), 'dfaebe28'],
         [path.join('smc', 'Headered', 'speed_test_v51.smc'), '9adca6cc'],
         [`${path.join('zip', 'Headered', 'fds_joypad_test.fds.zip')}|fds_joypad_test.fds`, '1e58456d'],
+        [`${path.join('zip', 'One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'],
         [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'],
         [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'],
         [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'],
@@ -326,7 +326,6 @@ describe('with explicit DATs', () => {
       expect(result.outputFilesAndCrcs).toEqual([
         [path.join('One', 'Fizzbuzz.nes'), '370517b5'],
         [path.join('One', 'Foobar.lnx'), 'b22c9747'],
-        [path.join('One', 'Lorem Ipsum.rom'), '70856527'],
         [path.join('One', 'One Three', 'One.rom'), 'f817a89f'],
         [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'],
         [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'],
@@ -365,7 +364,6 @@ describe('with explicit DATs', () => {
       expect(result.outputFilesAndCrcs).toEqual([
         [path.join('One', 'Fizzbuzz.nes'), '370517b5'],
         [path.join('One', 'Foobar.lnx'), 'b22c9747'],
-        [path.join('One', 'Lorem Ipsum.rom'), '70856527'],
         [path.join('One', 'One Three', 'One.rom'), 'f817a89f'],
         [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'],
         [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'],
@@ -419,7 +417,7 @@ describe('with explicit DATs', () => {
         [path.join('igir combined', 'KDULVQN.rom'), 'b1c303e4'],
         [path.join('igir combined', 'LCDTestROM.lnx'), '2d251538'],
         [path.join('igir combined', 'LCDTestROM.lyx'), '42583855'],
-        [path.join('igir combined', 'Lorem Ipsum.rom'), '70856527'],
+        [`${path.join('igir combined', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'],
         [path.join('igir combined', 'One Three', 'One.rom'), 'f817a89f'],
         [path.join('igir combined', 'One Three', 'Three.rom'), 'ff46c5d8'],
         [path.join('igir combined', 'speed_test_v51.sfc'), '8beffd94'],
@@ -453,6 +451,7 @@ describe('with explicit DATs', () => {
         path.join('raw', 'loremipsum.rom'),
         path.join('raw', 'one.rom'),
         path.join('raw', 'three.rom'),
+        path.join('zip', 'loremipsum.zip'),
       ]);
       expect(result.cleanedFiles).toHaveLength(0);
     });
@@ -554,7 +553,7 @@ describe('with explicit DATs', () => {
         [`${path.join('Headerless', 'speed_test_v51.zip')}|speed_test_v51.sfc`, '8beffd94'],
         [`${path.join('One', 'Fizzbuzz.zip')}|Fizzbuzz.nes`, '370517b5'],
         [`${path.join('One', 'Foobar.zip')}|Foobar.lnx`, 'b22c9747'],
-        [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.rom`, '70856527'],
+        [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.zip`, '7ee77289'],
         [`${path.join('One', 'One Three.zip')}|One.rom`, 'f817a89f'],
         [`${path.join('One', 'One Three.zip')}|Three.rom`, 'ff46c5d8'],
         [`${path.join('One', 'Three Four Five.zip')}|Five.rom`, '3e5daf67'],
@@ -613,7 +612,7 @@ describe('with explicit DATs', () => {
         ['Headerless.zip|speed_test_v51.sfc', '8beffd94'],
         ['One.zip|Fizzbuzz.nes', '370517b5'],
         ['One.zip|Foobar.lnx', 'b22c9747'],
-        ['One.zip|Lorem Ipsum.rom', '70856527'],
+        ['One.zip|Lorem Ipsum.zip', '7ee77289'],
         [`One.zip|${path.join('One Three', 'One.rom')}`, 'f817a89f'],
         [`One.zip|${path.join('One Three', 'Three.rom')}`, 'ff46c5d8'],
         [`One.zip|${path.join('Three Four Five', 'Five.rom')}`, '3e5daf67'],
@@ -658,7 +657,7 @@ describe('with explicit DATs', () => {
         [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc -> ${path.join('<input>', 'headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'],
         [`${path.join('One', 'Fizzbuzz.nes')} -> ${path.join('<input>', 'raw', 'fizzbuzz.nes')}`, '370517b5'],
         [`${path.join('One', 'Foobar.lnx')} -> ${path.join('<input>', 'foobar.lnx')}`, 'b22c9747'],
-        [`${path.join('One', 'Lorem Ipsum.rom')} -> ${path.join('<input>', 'raw', 'loremipsum.rom')}`, '70856527'],
+        [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom -> ${path.join('<input>', 'zip', 'loremipsum.zip')}|loremipsum.rom`, '70856527'],
         [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')} -> ${path.join('<input>', 'zip', 'onetwothree.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'],
         [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')} -> ${path.join('<input>', 'zip', 'onetwothree.zip')}|${path.join('2', 'two.rom')}`, '96170874'],
         [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')} -> ${path.join('<input>', 'zip', 'onetwothree.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'],
@@ -715,7 +714,7 @@ describe('with explicit DATs', () => {
         [path.join('Headerless', 'speed_test_v51.sfc'), '8beffd94'],
         [path.join('One', 'Fizzbuzz.nes'), '370517b5'],
         [path.join('One', 'Foobar.lnx'), 'b22c9747'],
-        [path.join('One', 'Lorem Ipsum.rom'), '70856527'],
+        [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'],
         [path.join('One', 'One Three', 'One.rom'), 'f817a89f'],
         [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'],
         [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'],

test/modules/romScanner.test.ts

@@ -4,7 +4,8 @@ import path from 'node:path';
 import Constants from '../../src/constants.js';
 import ROMScanner from '../../src/modules/romScanner.js';
 import fsPoly from '../../src/polyfill/fsPoly.js';
-import Options from '../../src/types/options.js';
+import { ChecksumBitmask } from '../../src/types/files/fileChecksums.js';
+import Options, { OptionsProps } from '../../src/types/options.js';
 import ProgressBarFake from '../console/progressBarFake.js';
 
 function createRomScanner(input: string[], inputExclude: string[] = []): ROMScanner {
@@ -40,6 +41,18 @@ describe('multiple files', () => {
     await expect(createRomScanner(['test/fixtures/roms/**/*', 'test/fixtures/roms/**/*.{rom,zip}']).scan()).resolves.toHaveLength(expectedRomFiles);
   });
 
+  test.each([
+    [{ input: ['test/fixtures/roms'] }, 90],
+    [{ input: ['test/fixtures/roms/7z'] }, 12],
+    [{ input: ['test/fixtures/roms/rar'] }, 12],
+    [{ input: ['test/fixtures/roms/tar'] }, 12],
+    [{ input: ['test/fixtures/roms/zip'] }, 15],
+  ] satisfies [OptionsProps, number][])('should calculate checksums of archives: %s', async (optionsProps, expectedRomFiles) => {
+    const scannedFiles = await new ROMScanner(new Options(optionsProps), new ProgressBarFake())
+      .scan(ChecksumBitmask.CRC32, true);
+    expect(scannedFiles).toHaveLength(expectedRomFiles);
+  });
+
   it('should scan multiple files with some file exclusions', async () => {
     await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44);
     await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom', 'test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44);
@@ -169,8 +182,10 @@ describe('multiple files', () => {
   });
 });
 
-it('should scan single files', async () => {
-  await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1);
-  await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1);
-  await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1);
+describe('single files', () => {
+  it('should scan single files with no exclusions', async () => {
+    await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1);
+    await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1);
+    await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1);
+  });
 });