From 92e2493882a8df0408b67116210b3404c98bc1d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Sj=C3=B6blom?= Date: Sat, 6 Jun 2026 10:08:24 +0200 Subject: [PATCH 1/2] Adding compact subcommand --- docs/commands/compact.md | 20 +++++++ src/commands.cpp | 11 ++++ src/commands.h | 1 + src/main.cpp | 14 ++++- src/mpq.cpp | 14 +++++ src/mpq.h | 1 + test/test_compact.py | 112 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 docs/commands/compact.md create mode 100644 test/test_compact.py diff --git a/docs/commands/compact.md b/docs/commands/compact.md new file mode 100644 index 0000000..3ebe6d0 --- /dev/null +++ b/docs/commands/compact.md @@ -0,0 +1,20 @@ +# compact + +Compacts an MPQ archive. It performs a complete archive rebuild, effectively defragmenting the MPQ archive, removing all gaps that have been created by adding, replacing, renaming or deleting files within the archive. To succeed, the function requires all files in MPQ archive to be accessible and their filenames to be known. + +This may take several minutes to complete for large archives. + +```bash +$ mpqcli compact wow-patch.mpq +[*] Compacting archive. This may take some time... +``` + + +## Use an external listfile + +The compact command requires all files inside the archive to be known. Older MPQ archives do not contain (complete) file paths of their content. By using the `-l` or `--listfile` argument, one can provide an external listfile that lists the content of the MPQ archive. Listfiles can be downloaded on [Ladislav Zezula's site](http://www.zezula.net/en/mpq/download.html). + +```bash +$ mpqcli compact -l /path/to/listfile DIABDAT.MPQ +[*] Compacting archive. This may take some time... +``` diff --git a/src/commands.cpp b/src/commands.cpp index c100a5b..75cfa71 100644 --- a/src/commands.cpp +++ b/src/commands.cpp @@ -338,3 +338,14 @@ int HandleVerify(const std::string &target, bool printSignature) { CloseMpqArchive(hArchive); return result; } + +int HandleCompact(const std::string &target, const std::optional &listfileName) { + HANDLE hArchive; + if (!OpenMpqArchive(target, &hArchive, 0)) { + return 1; + } + + const int result = CompactMpqArchive(hArchive, listfileName); + CloseMpqArchive(hArchive); + return result; +} diff --git a/src/commands.h b/src/commands.h index 3ba0d20..277b513 100644 --- a/src/commands.h +++ b/src/commands.h @@ -32,5 +32,6 @@ int HandleExtract(const std::string &target, const std::optional &o int HandleRead(const std::string &file, const std::string &target, const std::optional &locale); int HandleVerify(const std::string &target, bool printSignature); +int HandleCompact(const std::string &target, const std::optional &listfileName); #endif // COMMANDS_H diff --git a/src/main.cpp b/src/main.cpp index e0d1b99..d3c83e4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,7 +25,7 @@ int main(int argc, char **argv) { std::optional baseLocale; // add, create, extract, read, remove std::optional basePath; // add, create std::optional baseOutput; // create, extract - std::optional baseListfileName; // extract, list + std::optional baseListfileName; // extract, list, compact std::optional baseGameProfile; // add, create int64_t fileDwFlags = -1; // add, create int64_t fileDwCompression = -1; // add, create @@ -213,6 +213,14 @@ int main(int argc, char **argv) { ->check(CLI::ExistingFile); verify->add_flag("-p,--print", verifyPrintSignature, "Print the digital signature (in hex)"); + // Subcommand: compact + CLI::App *compact = app.add_subcommand("compact", "Compact the MPQ archive"); + compact->add_option("target", baseTarget, "Target MPQ archive") + ->required() + ->check(CLI::ExistingFile); + compact->add_option("-l,--listfile", baseListfileName, "File listing content of an MPQ archive") + ->check(CLI::ExistingFile); + try { app.parse(argc, argv); } catch (const CLI::ParseError &e) { @@ -295,5 +303,9 @@ int main(int argc, char **argv) { return HandleVerify(baseTarget, verifyPrintSignature); } + if (app.got_subcommand(compact)) { + return HandleCompact(baseTarget, baseListfileName); + } + return 0; } diff --git a/src/mpq.cpp b/src/mpq.cpp index db50424..140de9b 100644 --- a/src/mpq.cpp +++ b/src/mpq.cpp @@ -651,6 +651,20 @@ uint32_t VerifyMpqArchive(HANDLE hArchive) { return SFileVerifyArchive(hArchive); } +int CompactMpqArchive(HANDLE hArchive, const std::optional &listfileName) { + std::cout << "[*] Compacting archive. This may take some time..." << std::endl; + // Check if the user provided a listfile input + const char *listfile = listfileName.has_value() ? listfileName->c_str() : nullptr; + + if (!SFileCompactArchive(hArchive, listfile, false)) { + const auto error = SErrGetLastError(); + std::cerr << "[!] Failed to compact archive: (" << error << ") " << StormErrorString(error) + << std::endl; + return 1; + } + return 0; +} + int32_t PrintMpqSignature(HANDLE hArchive, const std::string &target) { // Determine if we have a strong or weak digital signature int32_t signatureType = GetFileInfo(hArchive, SFileMpqSignatures); diff --git a/src/mpq.h b/src/mpq.h index 234360d..27d7f92 100644 --- a/src/mpq.h +++ b/src/mpq.h @@ -36,6 +36,7 @@ std::unique_ptr ReadFile(HANDLE hArchive, const char *szFileName, unsign LCID preferredLocale); void PrintMpqInfo(HANDLE hArchive, const std::optional &infoProperty); uint32_t VerifyMpqArchive(HANDLE hArchive); +int CompactMpqArchive(HANDLE hArchive, const std::optional &listfileName); int32_t PrintMpqSignature(HANDLE hArchive, const std::string &target); template diff --git a/test/test_compact.py b/test/test_compact.py new file mode 100644 index 0000000..63481f8 --- /dev/null +++ b/test/test_compact.py @@ -0,0 +1,112 @@ +import random +import shutil +from pathlib import Path +import subprocess + + +def test_compact_nonexistent_file(binary_path): + """ + Test compacting non-existent MPQ file. + + This test checks: + - If the application exits correctly. + """ + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "non_existent_file.mpq" + + result = subprocess.run( + [str(binary_path), "compact", str(test_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + assert result.returncode != 1, f"mpqcli failed with error: {result.stderr}" + + +def test_compact_nonmpq_file(binary_path): + """ + Test compacting illegal MPQ file. + + This test checks: + - If the application exits correctly. + """ + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "cats.txt" + + expected_prefix = "[!] Failed to open MPQ archive" + + result = subprocess.run( + [str(binary_path), "compact", str(test_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output_lines = result.stderr.splitlines() + + assert result.returncode == 1, f"mpqcli failed with error: {result.stderr}" + assert len(output_lines) == 1, f"Unexpected output: {output_lines}" + assert output_lines[0].startswith(expected_prefix), f"Unexpected output: {output_lines}" + + +def test_compact_file(binary_path): + """ + Test compacting an MPQ archive. + + This test checks: + - Creating an archive with a small file and a ~512 KB incompressible file + - Removing the larger file from the archive + - Compacting the archive + - Verifying that the archive size shrinks significantly afterwards + """ + script_dir = Path(__file__).parent + data_dir = script_dir / "data" + + compaction_files_dir = data_dir / "compaction_files" + shutil.rmtree(compaction_files_dir, ignore_errors=True) + compaction_files_dir.mkdir(parents=True, exist_ok=True) + + mpq_file = data_dir / "mpq_for_compaction.mpq" + mpq_file.unlink(missing_ok=True) + + small_file = compaction_files_dir / "small.txt" + small_file.write_text("This is a small text file.\n", newline="\n") + + large_file_name = "large.bin" + large_file = compaction_files_dir / large_file_name + large_file_size = 512 * 1024 + large_file.write_bytes(random.Random(0).randbytes(large_file_size)) + + result = subprocess.run( + [str(binary_path), "create", "-o", str(mpq_file), str(compaction_files_dir)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + assert result.returncode == 0, f"create failed: {result.stderr}" + + result = subprocess.run( + [str(binary_path), "remove", str(mpq_file), large_file_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + assert result.returncode == 0, f"remove failed: {result.stderr}" + + size_before = mpq_file.stat().st_size + + result = subprocess.run( + [str(binary_path), "compact", str(mpq_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + assert result.returncode == 0, f"compact failed: {result.stderr}" + + size_after = mpq_file.stat().st_size + + assert size_before - size_after > size_before / 2, ( + f"Expected compaction to shrink the archive significantly, " + f"but size went from {size_before} bytes to {size_after} bytes" + ) From 62d06a3934c09e4a14c358e99c6858b8b4d56be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Sj=C3=B6blom?= Date: Sat, 6 Jun 2026 11:19:58 +0200 Subject: [PATCH 2/2] Adding more test cases for compact subcommand --- test/test_compact.py | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/test_compact.py b/test/test_compact.py index 63481f8..575fa41 100644 --- a/test/test_compact.py +++ b/test/test_compact.py @@ -50,6 +50,63 @@ def test_compact_nonmpq_file(binary_path): assert output_lines[0].startswith(expected_prefix), f"Unexpected output: {output_lines}" +def test_compact_mpq_without_listfile(binary_path, generate_mpq_without_internal_listfile): + """ + Test compacting MPQ file with no internal listfile, + and no externally provided one. + + This test checks: + - That the application exits correctly and prints error message. + """ + _ = generate_mpq_without_internal_listfile + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "mpq_without_internal_listfile.mpq" + + expected_prefix = "[!] Failed to compact archive: (10007) At least one file name is unknown (listfile is incomplete)" + + result = subprocess.run( + [str(binary_path), "compact", str(test_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output_lines = result.stderr.splitlines() + + assert result.returncode == 1, f"mpqcli failed with error: {result.stderr}" + assert len(output_lines) == 1, f"Unexpected output: {output_lines}" + assert output_lines[0].startswith(expected_prefix), f"Unexpected output: {output_lines}" + + +def test_compact_mpq_with_listfile(binary_path, generate_mpq_without_internal_listfile): + """ + Test compacting MPQ file with no internal listfile, + instead providing an external one. + + This test checks: + - That compaction works when the user provides an external listfile. + """ + _ = generate_mpq_without_internal_listfile + script_dir = Path(__file__).parent + test_file = script_dir / "data" / "mpq_without_internal_listfile.mpq" + listfile = script_dir / "data" / "listfile.txt" + listfile.write_text("cats.txt\ndogs.txt\ncapybaras.txt") + + expected_output = ["[*] Compacting archive. This may take some time..."] + + result = subprocess.run( + [str(binary_path), "compact", str(test_file), "--listfile", str(listfile)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output_lines = result.stdout.splitlines() + + assert result.returncode == 0, f"mpqcli failed with error: {result.stderr}" + assert output_lines == expected_output, f"Unexpected output: {output_lines}" + + def test_compact_file(binary_path): """ Test compacting an MPQ archive.