From 227b387f0f7375ad43c568275b6bca797ab8bafb Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 16:12:00 -0500 Subject: [PATCH 01/10] Add external listfile functionality. --- .../systems/crigges/jmpq3/JMpqEditor.java | 68 +++++++++++++++---- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index bbbe910..6961d00 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -209,6 +209,40 @@ private void readAttributesFile() { } } + /** + * For use when the MPQ is missing a (listfile) + * Adds this custom listfile into the MPQ and uses it + * for rebuilding purposes. + * If this is not a full listfile, the end result will be missing files. + * + * @param externalListfilePath Path to a file containing listfile entries + */ + public void addExternalListfile(File externalListfilePath) { + if(!externalListfilePath.exists()) { + log.warn("External MPQ File: " + externalListfilePath.getAbsolutePath() + + " does not exist and will not be used"); + return; + } + try { + // Copy the user's file to our temp directory + File tempFile = File.createTempFile("list", "file", JMpqEditor.tempDir); + tempFile.deleteOnExit(); + Files.copy(externalListfilePath.toPath(), tempFile.toPath()); + // Read and apply listfile + listFile = new Listfile(Files.readAllBytes(tempFile.toPath())); + checkListfileEntries(); + // Operation succeeded and added a listfile so we can now write properly. + canWrite = true; + } catch (Exception ex) { + log.warn("Could not apply external listfile: " + externalListfilePath.getAbsolutePath()); + canWrite = false; + } + } + + /** + * Reads an internal Listfile name called (listfile) + * and applies that as the archive's listfile. + */ private void readListFile() { if (hasFile("(listfile)")) { try { @@ -216,18 +250,7 @@ private void readListFile() { tempFile.deleteOnExit(); extractFile("(listfile)", tempFile); listFile = new Listfile(Files.readAllBytes(tempFile.toPath())); - int hiddenFiles = (hasFile("(attributes)") ? 2 : 1) + (hasFile("(signature)") ? 1 : 0); - if (canWrite) { - if (listFile.getFiles().size() >= blockTable.getAllVaildBlocks().size() - hiddenFiles) { - log.warn("mpq's listfile is incomplete. Blocks without listfile entry will be discarded"); - } - for (String fileName : listFile.getFiles()) { - if (!hasFile(fileName)) { - log.warn("listfile entry does not exist in archive and will be discarded: " + fileName); - } - } - listFile.getFileMap().entrySet().removeIf(file -> !hasFile(file.getValue())); - } + checkListfileEntries(); } catch (Exception e) { log.warn("Extracting the mpq's listfile failed. It cannot be rebuild.", e); } @@ -237,6 +260,27 @@ private void readListFile() { } } + /** + * Performs verification to see if we know all the blocks of this file. + * Prints warnings if we don't know all blocks. + * + * @throws JMpqException If retrieving valid blocks fails + */ + private void checkListfileEntries() throws JMpqException { + int hiddenFiles = (hasFile("(attributes)") ? 2 : 1) + (hasFile("(signature)") ? 1 : 0); + if (canWrite) { + if (listFile.getFiles().size() >= blockTable.getAllVaildBlocks().size() - hiddenFiles) { + log.warn("mpq's listfile is incomplete. Blocks without listfile entry will be discarded"); + } + for (String fileName : listFile.getFiles()) { + if (!hasFile(fileName)) { + log.warn("listfile entry does not exist in archive and will be discarded: " + fileName); + } + } + listFile.getFileMap().entrySet().removeIf(file -> !hasFile(file.getValue())); + } + } + private void readBlockTable() throws IOException { ByteBuffer blockBuffer = ByteBuffer.allocate(blockSize * 16).order(ByteOrder.LITTLE_ENDIAN); fc.position(headerOffset + blockPos); From b7906b8933c89a07b149d4b06251aa4cdca6a1af Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 16:27:57 -0500 Subject: [PATCH 02/10] Don't create temp file --- src/main/java/systems/crigges/jmpq3/JMpqEditor.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index 6961d00..aa99f97 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -224,12 +224,8 @@ public void addExternalListfile(File externalListfilePath) { return; } try { - // Copy the user's file to our temp directory - File tempFile = File.createTempFile("list", "file", JMpqEditor.tempDir); - tempFile.deleteOnExit(); - Files.copy(externalListfilePath.toPath(), tempFile.toPath()); // Read and apply listfile - listFile = new Listfile(Files.readAllBytes(tempFile.toPath())); + listFile = new Listfile(Files.readAllBytes(externalListfilePath.toPath())); checkListfileEntries(); // Operation succeeded and added a listfile so we can now write properly. canWrite = true; From 65168c550b5f7c1be7c377ad1270ae04bf93a5ac Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 16:38:00 -0500 Subject: [PATCH 03/10] Remove missing entries from listfile to avoid crashing close method. --- .../java/systems/crigges/jmpq3/JMpqEditor.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index aa99f97..f41bb2e 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -21,6 +21,7 @@ import java.nio.channels.WritableByteChannel; import java.nio.file.*; import java.util.*; +import java.util.function.Predicate; import static systems.crigges.jmpq3.MpqFile.*; @@ -227,6 +228,7 @@ public void addExternalListfile(File externalListfilePath) { // Read and apply listfile listFile = new Listfile(Files.readAllBytes(externalListfilePath.toPath())); checkListfileEntries(); + removeMissingFiles(); // Operation succeeded and added a listfile so we can now write properly. canWrite = true; } catch (Exception ex) { @@ -235,6 +237,19 @@ public void addExternalListfile(File externalListfilePath) { } } + /** + * Removes files from the listfile if they aren't + * actually in the map. + */ + private void removeMissingFiles() { + Iterator it = listFile.getFiles().iterator(); + while(it.hasNext()) { + if(!hasFile(it.next())) { + it.remove(); + } + } + } + /** * Reads an internal Listfile name called (listfile) * and applies that as the archive's listfile. From 3e7fa95e376f261feff32fc994ee94b15a7cba67 Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 16:41:15 -0500 Subject: [PATCH 04/10] Remove unnecessary import --- src/main/java/systems/crigges/jmpq3/JMpqEditor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index f41bb2e..98d65c3 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -21,7 +21,6 @@ import java.nio.channels.WritableByteChannel; import java.nio.file.*; import java.util.*; -import java.util.function.Predicate; import static systems.crigges.jmpq3.MpqFile.*; From 52c1bb774a01dae4234baa20cd9f6b9e8d9ffc2a Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 16:44:15 -0500 Subject: [PATCH 05/10] Add some logic for preserving original readOnly flag --- src/main/java/systems/crigges/jmpq3/JMpqEditor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index 98d65c3..5e91d28 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -120,6 +120,8 @@ public class JMpqEditor implements AutoCloseable { /** If write operations are supported on the archive. */ private boolean canWrite; + /** If the archive was originally read-only */ + private boolean openedAsReadOnly; /** * Creates a new MPQ editor for the MPQ file at the specified path. @@ -138,6 +140,7 @@ public class JMpqEditor implements AutoCloseable { public JMpqEditor(Path mpqArchive, MPQOpenOption... openOptions) throws JMpqException { // process open options canWrite = !Arrays.asList(openOptions).contains(MPQOpenOption.READ_ONLY); + openedAsReadOnly = !canWrite; legacyCompatibility = Arrays.asList(openOptions).contains(MPQOpenOption.FORCE_V0); log.debug(mpqArchive.toString()); try { @@ -229,10 +232,11 @@ public void addExternalListfile(File externalListfilePath) { checkListfileEntries(); removeMissingFiles(); // Operation succeeded and added a listfile so we can now write properly. - canWrite = true; + // (as long as it wasn't read-only to begin with) + canWrite = !openedAsReadOnly; } catch (Exception ex) { log.warn("Could not apply external listfile: " + externalListfilePath.getAbsolutePath()); - canWrite = false; + // The value of canWrite is not changed intentionally } } From 1a4fc49a03e781b5a1124c2ea55e22b781710bf6 Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 17:04:55 -0500 Subject: [PATCH 06/10] Add unit tests, correct some method names, refactor code --- .../systems/crigges/jmpq3/JMpqEditor.java | 37 ++++++++++++++----- .../systems/crigges/jmpq3test/MpqTests.java | 18 ++++++++- src/test/resources/listfile.txt | 6 +++ 3 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/listfile.txt diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index 5e91d28..a711cb7 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -220,7 +220,7 @@ private void readAttributesFile() { * * @param externalListfilePath Path to a file containing listfile entries */ - public void addExternalListfile(File externalListfilePath) { + public void setExternalListfile(File externalListfilePath) { if(!externalListfilePath.exists()) { log.warn("External MPQ File: " + externalListfilePath.getAbsolutePath() + " does not exist and will not be used"); @@ -283,16 +283,26 @@ private void readListFile() { private void checkListfileEntries() throws JMpqException { int hiddenFiles = (hasFile("(attributes)") ? 2 : 1) + (hasFile("(signature)") ? 1 : 0); if (canWrite) { - if (listFile.getFiles().size() >= blockTable.getAllVaildBlocks().size() - hiddenFiles) { - log.warn("mpq's listfile is incomplete. Blocks without listfile entry will be discarded"); - } - for (String fileName : listFile.getFiles()) { - if (!hasFile(fileName)) { - log.warn("listfile entry does not exist in archive and will be discarded: " + fileName); - } + checkListfileCompleteness(hiddenFiles); + } + } + + /** + * Checks listfile for completeness against block table + * + * @param hiddenFiles Num. hidden files + * @throws JMpqException If retrieving valid blocks fails + */ + private void checkListfileCompleteness(int hiddenFiles) throws JMpqException { + if (listFile.getFiles().size() >= blockTable.getAllVaildBlocks().size() - hiddenFiles) { + log.warn("mpq's listfile is incomplete. Blocks without listfile entry will be discarded"); + } + for (String fileName : listFile.getFiles()) { + if (!hasFile(fileName)) { + log.warn("listfile entry does not exist in archive and will be discarded: " + fileName); } - listFile.getFileMap().entrySet().removeIf(file -> !hasFile(file.getValue())); } + listFile.getFileMap().entrySet().removeIf(file -> !hasFile(file.getValue())); } private void readBlockTable() throws IOException { @@ -1104,4 +1114,13 @@ public String toString() { return "JMpqEditor [headerSize=" + headerSize + ", archiveSize=" + archiveSize + ", formatVersion=" + formatVersion + ", discBlockSize=" + discBlockSize + ", hashPos=" + hashPos + ", blockPos=" + blockPos + ", hashSize=" + hashSize + ", blockSize=" + blockSize + ", hashMap=" + hashTable + "]"; } + + /** + * Returns an unmodifiable collection of all Listfile entries + * + * @return Listfile entries + */ + public Collection getListfileEntries() { + return Collections.unmodifiableCollection(listFile.getFiles()); + } } diff --git a/src/test/java/systems/crigges/jmpq3test/MpqTests.java b/src/test/java/systems/crigges/jmpq3test/MpqTests.java index 4c3a089..3594211 100644 --- a/src/test/java/systems/crigges/jmpq3test/MpqTests.java +++ b/src/test/java/systems/crigges/jmpq3test/MpqTests.java @@ -20,8 +20,6 @@ import java.nio.file.StandardCopyOption; import java.util.*; -import static systems.crigges.jmpq3.HashTable.calculateFileKey; - /** * Created by Frotty on 06.03.2017. */ @@ -136,6 +134,22 @@ public void testRebuild() throws IOException { } } + @Test + public void testExternalListfile() throws Exception { + File mpq = getFile("mpqs/normalMap.w3x"); + File listFile = getFile("normalMap.w3x"); + JMpqEditor mpqEditor = new JMpqEditor(mpq, MPQOpenOption.FORCE_V0); + if(mpqEditor.isCanWrite()) { + mpqEditor.deleteFile("(listfile)"); + } + mpqEditor.setExternalListfile(listFile); + Assert.assertTrue(mpqEditor.getListfileEntries().contains("scripts\\war3map.j")); + Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.j")); + Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.w3u")); + Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.w3a")); + Assert.assertTrue(mpqEditor.getListfileEntries().contains("customFile.j")); + } + @Test public void testRecompressBuild() throws IOException { File[] mpqs = getMpqs(); diff --git a/src/test/resources/listfile.txt b/src/test/resources/listfile.txt new file mode 100644 index 0000000..89043f5 --- /dev/null +++ b/src/test/resources/listfile.txt @@ -0,0 +1,6 @@ +scripts\war3map.j +war3map.j +war3map.w3u +war3map.w3t +war3map.w3a +customFile.j \ No newline at end of file From 91c39ff81938eb1b59d5b50f6c0dcb14706c353b Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 2 May 2020 17:05:58 -0500 Subject: [PATCH 07/10] Simplify unit test --- src/test/java/systems/crigges/jmpq3test/MpqTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/systems/crigges/jmpq3test/MpqTests.java b/src/test/java/systems/crigges/jmpq3test/MpqTests.java index 3594211..b3e3c99 100644 --- a/src/test/java/systems/crigges/jmpq3test/MpqTests.java +++ b/src/test/java/systems/crigges/jmpq3test/MpqTests.java @@ -143,11 +143,7 @@ public void testExternalListfile() throws Exception { mpqEditor.deleteFile("(listfile)"); } mpqEditor.setExternalListfile(listFile); - Assert.assertTrue(mpqEditor.getListfileEntries().contains("scripts\\war3map.j")); - Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.j")); - Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.w3u")); Assert.assertTrue(mpqEditor.getListfileEntries().contains("war3map.w3a")); - Assert.assertTrue(mpqEditor.getListfileEntries().contains("customFile.j")); } @Test From 76cf05c1b1d2a82bcf5fad68f13c03844bcc2af1 Mon Sep 17 00:00:00 2001 From: "griggszm@msoe.edu" Date: Sun, 3 May 2020 11:38:56 -0500 Subject: [PATCH 08/10] Fix failing test --- src/test/java/systems/crigges/jmpq3test/MpqTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/systems/crigges/jmpq3test/MpqTests.java b/src/test/java/systems/crigges/jmpq3test/MpqTests.java index b3e3c99..6bba7ce 100644 --- a/src/test/java/systems/crigges/jmpq3test/MpqTests.java +++ b/src/test/java/systems/crigges/jmpq3test/MpqTests.java @@ -137,7 +137,7 @@ public void testRebuild() throws IOException { @Test public void testExternalListfile() throws Exception { File mpq = getFile("mpqs/normalMap.w3x"); - File listFile = getFile("normalMap.w3x"); + File listFile = getFile("listfile.txt"); JMpqEditor mpqEditor = new JMpqEditor(mpq, MPQOpenOption.FORCE_V0); if(mpqEditor.isCanWrite()) { mpqEditor.deleteFile("(listfile)"); From 1c314cfa73e6eba7bfa773ce2ebf02313f1480bc Mon Sep 17 00:00:00 2001 From: Zach Date: Sat, 16 May 2020 15:46:52 -0500 Subject: [PATCH 09/10] Delete unnecessary method for removing extra files --- .../java/systems/crigges/jmpq3/JMpqEditor.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index a711cb7..cb64525 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -230,7 +230,6 @@ public void setExternalListfile(File externalListfilePath) { // Read and apply listfile listFile = new Listfile(Files.readAllBytes(externalListfilePath.toPath())); checkListfileEntries(); - removeMissingFiles(); // Operation succeeded and added a listfile so we can now write properly. // (as long as it wasn't read-only to begin with) canWrite = !openedAsReadOnly; @@ -240,19 +239,6 @@ public void setExternalListfile(File externalListfilePath) { } } - /** - * Removes files from the listfile if they aren't - * actually in the map. - */ - private void removeMissingFiles() { - Iterator it = listFile.getFiles().iterator(); - while(it.hasNext()) { - if(!hasFile(it.next())) { - it.remove(); - } - } - } - /** * Reads an internal Listfile name called (listfile) * and applies that as the archive's listfile. From 5b6e5a8db2a9be327609b07582b85ec25f0a906f Mon Sep 17 00:00:00 2001 From: Frotty Date: Mon, 22 Mar 2021 11:51:25 +0100 Subject: [PATCH 10/10] fixes --- src/main/java/systems/crigges/jmpq3/JMpqEditor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java index cb64525..992eff9 100644 --- a/src/main/java/systems/crigges/jmpq3/JMpqEditor.java +++ b/src/main/java/systems/crigges/jmpq3/JMpqEditor.java @@ -120,8 +120,6 @@ public class JMpqEditor implements AutoCloseable { /** If write operations are supported on the archive. */ private boolean canWrite; - /** If the archive was originally read-only */ - private boolean openedAsReadOnly; /** * Creates a new MPQ editor for the MPQ file at the specified path. @@ -140,7 +138,6 @@ public class JMpqEditor implements AutoCloseable { public JMpqEditor(Path mpqArchive, MPQOpenOption... openOptions) throws JMpqException { // process open options canWrite = !Arrays.asList(openOptions).contains(MPQOpenOption.READ_ONLY); - openedAsReadOnly = !canWrite; legacyCompatibility = Arrays.asList(openOptions).contains(MPQOpenOption.FORCE_V0); log.debug(mpqArchive.toString()); try { @@ -221,6 +218,10 @@ private void readAttributesFile() { * @param externalListfilePath Path to a file containing listfile entries */ public void setExternalListfile(File externalListfilePath) { + if(!canWrite) { + log.warn("The mpq was opened as readonly, setting an external listfile will have no effect."); + return; + } if(!externalListfilePath.exists()) { log.warn("External MPQ File: " + externalListfilePath.getAbsolutePath() + " does not exist and will not be used"); @@ -232,7 +233,6 @@ public void setExternalListfile(File externalListfilePath) { checkListfileEntries(); // Operation succeeded and added a listfile so we can now write properly. // (as long as it wasn't read-only to begin with) - canWrite = !openedAsReadOnly; } catch (Exception ex) { log.warn("Could not apply external listfile: " + externalListfilePath.getAbsolutePath()); // The value of canWrite is not changed intentionally @@ -280,7 +280,7 @@ private void checkListfileEntries() throws JMpqException { * @throws JMpqException If retrieving valid blocks fails */ private void checkListfileCompleteness(int hiddenFiles) throws JMpqException { - if (listFile.getFiles().size() >= blockTable.getAllVaildBlocks().size() - hiddenFiles) { + if (listFile.getFiles().size() <= blockTable.getAllVaildBlocks().size() - hiddenFiles) { log.warn("mpq's listfile is incomplete. Blocks without listfile entry will be discarded"); } for (String fileName : listFile.getFiles()) {