From 18930fbb52e909a25dfa8e462ab93f2933941212 Mon Sep 17 00:00:00 2001 From: John Scancella Date: Sun, 22 Jul 2018 11:30:47 -0400 Subject: [PATCH 1/9] refs #124 renamed to make it more clear that we verify all manifests, not just payload manifest --- .../repository/bagit/verify/BagVerifier.java | 8 ++++---- ...oadVerifier.java => ManifestVerifier.java} | 19 +++++++++--------- .../bagit/verify/PayloadVerifierTest.java | 20 +++++++++---------- 3 files changed, 24 insertions(+), 23 deletions(-) rename src/main/java/gov/loc/repository/bagit/verify/{PayloadVerifier.java => ManifestVerifier.java} (92%) diff --git a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java index 835e06beb..95fd7a127 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java @@ -37,7 +37,7 @@ public final class BagVerifier implements AutoCloseable{ private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class); private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); - private final PayloadVerifier manifestVerifier; + private final ManifestVerifier manifestVerifier; private final ExecutorService executor; /** @@ -74,7 +74,7 @@ public BagVerifier(final ExecutorService executor){ * @param executor the thread pool to use when doing work */ public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ - manifestVerifier = new PayloadVerifier(nameMapping, executor); + manifestVerifier = new ManifestVerifier(nameMapping, executor); this.executor = executor; } @@ -209,14 +209,14 @@ public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion()); - manifestVerifier.verifyPayload(bag, ignoreHiddenFiles); + manifestVerifier.verifyManifests(bag, ignoreHiddenFiles); } public ExecutorService getExecutor() { return executor; } - public PayloadVerifier getManifestVerifier() { + public ManifestVerifier getManifestVerifier() { return manifestVerifier; } } diff --git a/src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java similarity index 92% rename from src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java rename to src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java index 2a7fd884c..8d073b1d3 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/PayloadVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java @@ -31,8 +31,8 @@ /** * Responsible for all things related to the manifest during verification. */ -public class PayloadVerifier implements AutoCloseable{ - private static final Logger logger = LoggerFactory.getLogger(PayloadVerifier.class); +public class ManifestVerifier implements AutoCloseable{ + private static final Logger logger = LoggerFactory.getLogger(ManifestVerifier.class); private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); private transient final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping; @@ -42,7 +42,7 @@ public class PayloadVerifier implements AutoCloseable{ * Create a PayloadVerifier using a cached thread pool and the * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping */ - public PayloadVerifier(){ + public ManifestVerifier(){ this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newCachedThreadPool()); } @@ -51,7 +51,7 @@ public PayloadVerifier(){ * * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm */ - public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) { + public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) { this(nameMapping, Executors.newCachedThreadPool()); } @@ -61,7 +61,7 @@ public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameM * * @param executor the thread pool to use when doing work */ - public PayloadVerifier(final ExecutorService executor) { + public ManifestVerifier(final ExecutorService executor) { this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), executor); } @@ -71,7 +71,7 @@ public PayloadVerifier(final ExecutorService executor) { * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm * @param executor the thread pool to use when doing work */ - public PayloadVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) { + public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping, final ExecutorService executor) { this.nameMapping = nameMapping; this.executor = executor; } @@ -83,11 +83,12 @@ public void close() throws SecurityException{ } /** - * Verify that all the files in the payload directory are listed in the manifest and - * all files listed in the manifests exist. + * Verify that all the files in the payload directory are listed in the payload manifest and + * all files listed in all manifests exist. * * @param bag the bag to check to check * @param ignoreHiddenFiles to ignore hidden files unless they are specifically listed in a manifest + * * @throws IOException if there is a problem reading a file * @throws MaliciousPathException the path in the manifest was specifically crafted to cause harm * @throws UnsupportedAlgorithmException if the algorithm used for the manifest is unsupported @@ -95,7 +96,7 @@ public void close() throws SecurityException{ * @throws FileNotInPayloadDirectoryException if a file is listed in a manifest but doesn't exist in the payload directory * @throws InterruptedException if a thread is interrupted while doing work */ - public void verifyPayload(final Bag bag, final boolean ignoreHiddenFiles) + public void verifyManifests(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException, FileNotInPayloadDirectoryException, InterruptedException { diff --git a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java b/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java index 3293b8a79..5c3a14138 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java +++ b/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java @@ -20,11 +20,11 @@ public class PayloadVerifierTest { private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI()); private BagReader reader = new BagReader(); - private PayloadVerifier sut; + private ManifestVerifier sut; @BeforeEach public void setup(){ - sut = new PayloadVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); + sut = new ManifestVerifier(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); } @Test @@ -32,11 +32,11 @@ public void testOtherConstructors() throws Exception { rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI()); Bag bag = reader.read(rootDir); - sut = new PayloadVerifier(); - sut.verifyPayload(bag, true); + sut = new ManifestVerifier(); + sut.verifyManifests(bag, true); - sut = new PayloadVerifier(Executors.newCachedThreadPool()); - sut.verifyPayload(bag, true); + sut = new ManifestVerifier(Executors.newCachedThreadPool()); + sut.verifyManifests(bag, true); } @Test @@ -45,7 +45,7 @@ public void testErrorWhenManifestListFileThatDoesntExist() throws Exception{ Bag bag = reader.read(rootDir); Assertions.assertThrows(FileNotInPayloadDirectoryException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } @Test @@ -54,7 +54,7 @@ public void testErrorWhenFileIsntInManifest() throws Exception{ Bag bag = reader.read(rootDir); Assertions.assertThrows(FileNotInManifestException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } @Test @@ -62,7 +62,7 @@ public void testBagWithTagFilesInPayloadIsValid() throws Exception{ rootDir = Paths.get(new File("src/test/resources/bags/v0_96/bag-with-tagfiles-in-payload-manifest").toURI()); Bag bag = reader.read(rootDir); - sut.verifyPayload(bag, true); + sut.verifyManifests(bag, true); } @Test @@ -70,6 +70,6 @@ public void testNotALlFilesListedInAllManifestsThrowsException() throws Exceptio Path bagDir = Paths.get(new File("src/test/resources/notAllFilesListedInAllManifestsBag").toURI()); Bag bag = reader.read(bagDir); Assertions.assertThrows(FileNotInManifestException.class, - () -> { sut.verifyPayload(bag, true); }); + () -> { sut.verifyManifests(bag, true); }); } } From 5a8bb98102a3ef461baab156c1a3a76d9990b907 Mon Sep 17 00:00:00 2001 From: John Scancella Date: Sun, 22 Jul 2018 11:39:29 -0400 Subject: [PATCH 2/9] refs #123 - fixed error message formatting --- .../exceptions/MissingPayloadDirectoryException.java | 8 ++++++-- .../loc/repository/bagit/verify/MandatoryVerifier.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java b/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java index 766b05aa7..f0258a3e5 100644 --- a/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java +++ b/src/main/java/gov/loc/repository/bagit/exceptions/MissingPayloadDirectoryException.java @@ -1,12 +1,16 @@ package gov.loc.repository.bagit.exceptions; +import java.nio.file.Path; + +import org.slf4j.helpers.MessageFormatter; + /** * The payload directory is a required file. This class represents the error if it is not found. */ public class MissingPayloadDirectoryException extends Exception { private static final long serialVersionUID = 1L; - public MissingPayloadDirectoryException(final String message){ - super(message); + public MissingPayloadDirectoryException(final String message, final Path path){ + super(MessageFormatter.format(message, path).getMessage()); } } diff --git a/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java index e73ed3b02..7d14d8357 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/MandatoryVerifier.java @@ -84,7 +84,7 @@ public static void checkPayloadDirectoryExists(final Bag bag) throws MissingPayl final Path dataDir = PathUtils.getDataDir(bag); if(!Files.exists(dataDir)){ - throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error")); + throw new MissingPayloadDirectoryException(messages.getString("file_should_exist_error"), dataDir); } } From f6249939154b22055d20a414a4e0e8b00b4d0691 Mon Sep 17 00:00:00 2001 From: John Scancella Date: Sun, 22 Jul 2018 12:06:59 -0400 Subject: [PATCH 3/9] refs #119 - fixed different case issue and added test to check for any issues in the future --- .../loc/repository/bagit/conformance/ManifestChecker.java | 5 +++-- .../repository/bagit/hash/StandardSupportedAlgorithms.java | 6 +++--- .../loc/repository/bagit/conformance/BagLinterTest.java | 7 +++++++ src/test/resources/bags/v1_0/bag/bag-info.txt | 3 +++ src/test/resources/bags/v1_0/bag/bagit.txt | 2 ++ src/test/resources/bags/v1_0/bag/data/foo.txt | 1 + src/test/resources/bags/v1_0/bag/manifest-sha512.txt | 1 + src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt | 3 +++ 8 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 src/test/resources/bags/v1_0/bag/bag-info.txt create mode 100644 src/test/resources/bags/v1_0/bag/bagit.txt create mode 100644 src/test/resources/bags/v1_0/bag/data/foo.txt create mode 100644 src/test/resources/bags/v1_0/bag/manifest-sha512.txt create mode 100644 src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt diff --git a/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java b/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java index af9438c26..b7247dc4e 100644 --- a/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java +++ b/src/main/java/gov/loc/repository/bagit/conformance/ManifestChecker.java @@ -123,9 +123,10 @@ private static void checkManifestPayload(final Path manifestFile, final Charset String path = parsePath(line); path = checkForManifestCreatedWithMD5SumTools(path, warnings, warningsToIgnore); - paths.add(path.toLowerCase()); checkForDifferentCase(path, paths, manifestFile, warnings, warningsToIgnore); + paths.add(path.toLowerCase()); + if(encoding.name().startsWith("UTF")){ checkNormalization(path, manifestFile.getParent(), warnings, warningsToIgnore); } @@ -256,7 +257,7 @@ static void checkAlgorthm(final String algorithm, final Set warnin warnings.add(BagitWarning.WEAK_CHECKSUM_ALGORITHM); } - else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA-512".equals(upperCaseAlg)){ + else if(!warningsToIgnore.contains(BagitWarning.NON_STANDARD_ALGORITHM) && !"SHA512".equals(upperCaseAlg)){ logger.warn(messages.getString("non_standard_algorithm_warning"), algorithm); warnings.add(BagitWarning.NON_STANDARD_ALGORITHM); } diff --git a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java index 40818316c..656485e89 100644 --- a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java +++ b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java @@ -6,9 +6,9 @@ public enum StandardSupportedAlgorithms implements SupportedAlgorithm{ MD5("MD5"), SHA1("SHA-1"), - SHA224("SHA-224"), - SHA256("SHA-256"), - SHA512("SHA-512"); + SHA224("SHA224"), + SHA256("SHA256"), + SHA512("SHA512"); private final String messageDigestName; diff --git a/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java b/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java index 3a9655fbb..830ca94a8 100644 --- a/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java +++ b/src/test/java/gov/loc/repository/bagit/conformance/BagLinterTest.java @@ -28,6 +28,13 @@ public void testClassIsWellDefined() throws NoSuchMethodException, InvocationTar assertUtilityClassWellDefined(BagLinter.class); } + @Test + public void testConformantBag() throws Exception{ + Path goodBag = Paths.get("src", "test", "resources", "bags", "v1_0", "bag"); + Set warnings = BagLinter.lintBag(goodBag); + Assertions.assertTrue(warnings.size() == 0); + } + @Test public void testLintBag() throws Exception{ Set expectedWarnings = new HashSet<>(); diff --git a/src/test/resources/bags/v1_0/bag/bag-info.txt b/src/test/resources/bags/v1_0/bag/bag-info.txt new file mode 100644 index 000000000..30723bdc8 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-22 +Payload-Oxum: 6.1 diff --git a/src/test/resources/bags/v1_0/bag/bagit.txt b/src/test/resources/bags/v1_0/bag/bagit.txt new file mode 100644 index 000000000..7bfcaecc4 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 1.0 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/bags/v1_0/bag/data/foo.txt b/src/test/resources/bags/v1_0/bag/data/foo.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/data/foo.txt @@ -0,0 +1 @@ +hello diff --git a/src/test/resources/bags/v1_0/bag/manifest-sha512.txt b/src/test/resources/bags/v1_0/bag/manifest-sha512.txt new file mode 100644 index 000000000..f3fd06d72 --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/manifest-sha512.txt @@ -0,0 +1 @@ +e7c22b994c59d9cf2b48e549b1e24666636045930d3da7c1acb299d1c3b7f931f94aae41edda2c2b207a36e10f8bcb8d45223e54878f5b316e7ce3b6bc019629 data/foo.txt diff --git a/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt b/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt new file mode 100644 index 000000000..732e7c45d --- /dev/null +++ b/src/test/resources/bags/v1_0/bag/tagmanifest-sha512.txt @@ -0,0 +1,3 @@ +f5bdbc7f273dd8b95d30b77b3d6f727d999ecbe9be06a7656388e7a3a46d963881563d779a2a99265b0c2785de2a7b72ac05fa7bc5b66b471d3f4e80fe9bb370 bag-info.txt +1d73ae108d4109b61f56698a5e19ee1f8947bdf8940bbce6adbe5e0940c2363caace6a547b4f1b3ec6a4fd2b7fa845e9cb9d28823bc72c59971718bb26f2fbd8 bagit.txt +35d40e38f5e2eb7261a1cc0c0ccf9c6c50d2e07e39a9460ec4f99046048f939d82647c68d8f744c653c7a37c05cc9d71153c8028786dfeba4322a3be5fe81af0 manifest-sha512.txt From d38a82a0052faba726d1436ab1bea79fa6607cec Mon Sep 17 00:00:00 2001 From: John Scancella Date: Sun, 22 Jul 2018 12:10:13 -0400 Subject: [PATCH 4/9] refs #121 - only use as many threads as CPUs --- .../java/gov/loc/repository/bagit/verify/BagVerifier.java | 4 ++-- .../gov/loc/repository/bagit/verify/ManifestVerifier.java | 4 ++-- .../repository/bagit/verify/CheckIfFileExistsTaskTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java index 95fd7a127..663cd8b8f 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java @@ -45,7 +45,7 @@ public final class BagVerifier implements AutoCloseable{ * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} */ public BagVerifier(){ - this(Executors.newCachedThreadPool(), new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); + this(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), new StandardBagitAlgorithmNameToSupportedAlgorithmMapping()); } /** @@ -54,7 +54,7 @@ public BagVerifier(){ * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm */ public BagVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){ - this(Executors.newCachedThreadPool(), nameMapping); + this(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()), nameMapping); } /** diff --git a/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java index 8d073b1d3..645953f15 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/ManifestVerifier.java @@ -43,7 +43,7 @@ public class ManifestVerifier implements AutoCloseable{ * {@link StandardBagitAlgorithmNameToSupportedAlgorithmMapping} mapping */ public ManifestVerifier(){ - this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newCachedThreadPool()); + this(new StandardBagitAlgorithmNameToSupportedAlgorithmMapping(), Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); } /** @@ -52,7 +52,7 @@ public ManifestVerifier(){ * @param nameMapping the mapping between BagIt algorithm name and the java supported algorithm */ public ManifestVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping) { - this(nameMapping, Executors.newCachedThreadPool()); + this(nameMapping, Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); } /** diff --git a/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java b/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java index eb0f1b7fb..6ea65d417 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java +++ b/src/test/java/gov/loc/repository/bagit/verify/CheckIfFileExistsTaskTest.java @@ -17,7 +17,7 @@ public class CheckIfFileExistsTaskTest extends TempFolderTest { @Test public void testNormalizedFileExists() throws Exception{ - ExecutorService executor = Executors.newCachedThreadPool(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); CountDownLatch latch = new CountDownLatch(1); Set missingFiles = new ConcurrentSkipListSet<>(); String filename = "Núñez.txt"; From f3c31145ec7b1566fd4b56209925f2ffda1db18a Mon Sep 17 00:00:00 2001 From: John Scancella Date: Tue, 24 Jul 2018 09:23:42 -0400 Subject: [PATCH 5/9] refs #119 - changed sha-1 to sha1 to be more inline with bagit-python. Also added tests for valid bags of each standard algorithm --- .../hash/StandardSupportedAlgorithms.java | 2 +- .../bagit/verify/BagVerifierTest.java | 35 +++++++++++++++++++ ...ierTest.java => ManifestVerifierTest.java} | 2 +- src/test/resources/md5Bag/bag-info.txt | 3 ++ src/test/resources/md5Bag/bagit.txt | 2 ++ src/test/resources/md5Bag/data/readme.txt | 1 + src/test/resources/md5Bag/manifest-md5.txt | 1 + src/test/resources/md5Bag/tagmanifest-md5.txt | 3 ++ src/test/resources/sha1Bag/bag-info.txt | 3 ++ src/test/resources/sha1Bag/bagit.txt | 2 ++ src/test/resources/sha1Bag/data/readme.txt | 1 + src/test/resources/sha1Bag/manifest-sha1.txt | 1 + .../resources/sha1Bag/tagmanifest-sha1.txt | 3 ++ src/test/resources/sha224Bag/bag-info.txt | 3 ++ src/test/resources/sha224Bag/bagit.txt | 2 ++ src/test/resources/sha224Bag/data/readme.txt | 1 + .../resources/sha224Bag/manifest-sha224.txt | 1 + .../sha224Bag/tagmanifest-sha224.txt | 3 ++ src/test/resources/sha256Bag/bag-info.txt | 3 ++ src/test/resources/sha256Bag/bagit.txt | 2 ++ src/test/resources/sha256Bag/data/readme.txt | 1 + .../resources/sha256Bag/manifest-sha256.txt | 1 + .../sha256Bag/tagmanifest-sha256.txt | 3 ++ src/test/resources/sha512Bag/bag-info.txt | 3 ++ src/test/resources/sha512Bag/bagit.txt | 2 ++ src/test/resources/sha512Bag/data/readme.txt | 1 + .../resources/sha512Bag/manifest-sha512.txt | 1 + .../sha512Bag/tagmanifest-sha512.txt | 3 ++ 28 files changed, 87 insertions(+), 2 deletions(-) rename src/test/java/gov/loc/repository/bagit/verify/{PayloadVerifierTest.java => ManifestVerifierTest.java} (98%) create mode 100644 src/test/resources/md5Bag/bag-info.txt create mode 100644 src/test/resources/md5Bag/bagit.txt create mode 100644 src/test/resources/md5Bag/data/readme.txt create mode 100644 src/test/resources/md5Bag/manifest-md5.txt create mode 100644 src/test/resources/md5Bag/tagmanifest-md5.txt create mode 100644 src/test/resources/sha1Bag/bag-info.txt create mode 100644 src/test/resources/sha1Bag/bagit.txt create mode 100644 src/test/resources/sha1Bag/data/readme.txt create mode 100644 src/test/resources/sha1Bag/manifest-sha1.txt create mode 100644 src/test/resources/sha1Bag/tagmanifest-sha1.txt create mode 100644 src/test/resources/sha224Bag/bag-info.txt create mode 100644 src/test/resources/sha224Bag/bagit.txt create mode 100644 src/test/resources/sha224Bag/data/readme.txt create mode 100644 src/test/resources/sha224Bag/manifest-sha224.txt create mode 100644 src/test/resources/sha224Bag/tagmanifest-sha224.txt create mode 100644 src/test/resources/sha256Bag/bag-info.txt create mode 100644 src/test/resources/sha256Bag/bagit.txt create mode 100644 src/test/resources/sha256Bag/data/readme.txt create mode 100644 src/test/resources/sha256Bag/manifest-sha256.txt create mode 100644 src/test/resources/sha256Bag/tagmanifest-sha256.txt create mode 100644 src/test/resources/sha512Bag/bag-info.txt create mode 100644 src/test/resources/sha512Bag/bagit.txt create mode 100644 src/test/resources/sha512Bag/data/readme.txt create mode 100644 src/test/resources/sha512Bag/manifest-sha512.txt create mode 100644 src/test/resources/sha512Bag/tagmanifest-sha512.txt diff --git a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java index 656485e89..32df24a7e 100644 --- a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java +++ b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java @@ -5,7 +5,7 @@ */ public enum StandardSupportedAlgorithms implements SupportedAlgorithm{ MD5("MD5"), - SHA1("SHA-1"), + SHA1("SHA1"), SHA224("SHA224"), SHA256("SHA256"), SHA512("SHA512"); diff --git a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java b/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java index 303e0acae..68955a149 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java +++ b/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java @@ -43,6 +43,41 @@ public void testStandardSupportedAlgorithms() throws Exception{ } } + @Test + public void testMD5Bag() throws Exception{ + Path bagDir = Paths.get("src", "test", "resources", "md5Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA1Bag() throws Exception{ + Path bagDir = Paths.get("src", "test", "resources", "sha1Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA224Bag() throws Exception{ + Path bagDir = Paths.get("src", "test", "resources", "sha224Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA256Bag() throws Exception{ + Path bagDir = Paths.get("src", "test", "resources", "sha256Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + + @Test + public void testSHA512Bag() throws Exception{ + Path bagDir = Paths.get("src", "test", "resources", "sha512Bag"); + Bag bag = reader.read(bagDir); + sut.isValid(bag, true); + } + @Test public void testVersion0_97IsValid() throws Exception{ Bag bag = reader.read(rootDir); diff --git a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java b/src/test/java/gov/loc/repository/bagit/verify/ManifestVerifierTest.java similarity index 98% rename from src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java rename to src/test/java/gov/loc/repository/bagit/verify/ManifestVerifierTest.java index 5c3a14138..5d10736bf 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/PayloadVerifierTest.java +++ b/src/test/java/gov/loc/repository/bagit/verify/ManifestVerifierTest.java @@ -15,7 +15,7 @@ import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping; import gov.loc.repository.bagit.reader.BagReader; -public class PayloadVerifierTest { +public class ManifestVerifierTest { private Path rootDir = Paths.get(new File("src/test/resources/bags/v0_97/bag").toURI()); private BagReader reader = new BagReader(); diff --git a/src/test/resources/md5Bag/bag-info.txt b/src/test/resources/md5Bag/bag-info.txt new file mode 100644 index 000000000..7dd5e14ce --- /dev/null +++ b/src/test/resources/md5Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 52.1 diff --git a/src/test/resources/md5Bag/bagit.txt b/src/test/resources/md5Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/md5Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/md5Bag/data/readme.txt b/src/test/resources/md5Bag/data/readme.txt new file mode 100644 index 000000000..7ec7b2d61 --- /dev/null +++ b/src/test/resources/md5Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid md5 bag used for testing valid bags diff --git a/src/test/resources/md5Bag/manifest-md5.txt b/src/test/resources/md5Bag/manifest-md5.txt new file mode 100644 index 000000000..62cd3ebf9 --- /dev/null +++ b/src/test/resources/md5Bag/manifest-md5.txt @@ -0,0 +1 @@ +aee452eebfbd978228775bf7b0e808dc data/readme.txt diff --git a/src/test/resources/md5Bag/tagmanifest-md5.txt b/src/test/resources/md5Bag/tagmanifest-md5.txt new file mode 100644 index 000000000..6201d4ecf --- /dev/null +++ b/src/test/resources/md5Bag/tagmanifest-md5.txt @@ -0,0 +1,3 @@ +59044a83bd40ec50b7ee927bf1e672a9 bag-info.txt +9e5ad981e0d29adc278f6a294b8c2aca bagit.txt +f1578aeea7f57af650f0a94565e40ceb manifest-md5.txt diff --git a/src/test/resources/sha1Bag/bag-info.txt b/src/test/resources/sha1Bag/bag-info.txt new file mode 100644 index 000000000..30ce5278d --- /dev/null +++ b/src/test/resources/sha1Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 43.1 diff --git a/src/test/resources/sha1Bag/bagit.txt b/src/test/resources/sha1Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha1Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha1Bag/data/readme.txt b/src/test/resources/sha1Bag/data/readme.txt new file mode 100644 index 000000000..0979f66d0 --- /dev/null +++ b/src/test/resources/sha1Bag/data/readme.txt @@ -0,0 +1 @@ +this is a sha1 bag used to test valid bags diff --git a/src/test/resources/sha1Bag/manifest-sha1.txt b/src/test/resources/sha1Bag/manifest-sha1.txt new file mode 100644 index 000000000..04bda92f9 --- /dev/null +++ b/src/test/resources/sha1Bag/manifest-sha1.txt @@ -0,0 +1 @@ +2f0973b52379ea88967e5e319941ff167d680ee9 data/readme.txt diff --git a/src/test/resources/sha1Bag/tagmanifest-sha1.txt b/src/test/resources/sha1Bag/tagmanifest-sha1.txt new file mode 100644 index 000000000..f4d7d5534 --- /dev/null +++ b/src/test/resources/sha1Bag/tagmanifest-sha1.txt @@ -0,0 +1,3 @@ +27bfaef124e06412f40c75bf6e5aa1cf4dbb9774 bag-info.txt +e2924b081506bac23f5fffe650ad1848a1c8ac1d bagit.txt +54adcabf9b1038d090d145d9315070c5493db1e6 manifest-sha1.txt diff --git a/src/test/resources/sha224Bag/bag-info.txt b/src/test/resources/sha224Bag/bag-info.txt new file mode 100644 index 000000000..40c3b6f85 --- /dev/null +++ b/src/test/resources/sha224Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 45.1 diff --git a/src/test/resources/sha224Bag/bagit.txt b/src/test/resources/sha224Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha224Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha224Bag/data/readme.txt b/src/test/resources/sha224Bag/data/readme.txt new file mode 100644 index 000000000..e05de9a0a --- /dev/null +++ b/src/test/resources/sha224Bag/data/readme.txt @@ -0,0 +1 @@ +this is a sha224 valid bag, used for testing diff --git a/src/test/resources/sha224Bag/manifest-sha224.txt b/src/test/resources/sha224Bag/manifest-sha224.txt new file mode 100644 index 000000000..a8ea47300 --- /dev/null +++ b/src/test/resources/sha224Bag/manifest-sha224.txt @@ -0,0 +1 @@ +476ba7f2f92170790679502b1f1bad48a9d0ae74b561cf558370a5fb data/readme.txt diff --git a/src/test/resources/sha224Bag/tagmanifest-sha224.txt b/src/test/resources/sha224Bag/tagmanifest-sha224.txt new file mode 100644 index 000000000..cba870c7c --- /dev/null +++ b/src/test/resources/sha224Bag/tagmanifest-sha224.txt @@ -0,0 +1,3 @@ +34458393a69d3c6a5ec82743e1817032b3dc875ae10ff19b2c55ed1c bag-info.txt +fe7e72d15c56a8e8017246f52765a93ce4823d1fa3ed1e0d1ac3695f bagit.txt +8c717410c037477322cdd4069feeff2aa09eb0991989807724defaae manifest-sha224.txt diff --git a/src/test/resources/sha256Bag/bag-info.txt b/src/test/resources/sha256Bag/bag-info.txt new file mode 100644 index 000000000..ec9f2e4da --- /dev/null +++ b/src/test/resources/sha256Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 44.1 diff --git a/src/test/resources/sha256Bag/bagit.txt b/src/test/resources/sha256Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha256Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha256Bag/data/readme.txt b/src/test/resources/sha256Bag/data/readme.txt new file mode 100644 index 000000000..4e8750d05 --- /dev/null +++ b/src/test/resources/sha256Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid sha256 bag used for testing diff --git a/src/test/resources/sha256Bag/manifest-sha256.txt b/src/test/resources/sha256Bag/manifest-sha256.txt new file mode 100644 index 000000000..f19ae2b49 --- /dev/null +++ b/src/test/resources/sha256Bag/manifest-sha256.txt @@ -0,0 +1 @@ +694d56447888829536b5782536f2bbe5c83325b1b2fd56a5ca676c991f196bf7 data/readme.txt diff --git a/src/test/resources/sha256Bag/tagmanifest-sha256.txt b/src/test/resources/sha256Bag/tagmanifest-sha256.txt new file mode 100644 index 000000000..c37479e35 --- /dev/null +++ b/src/test/resources/sha256Bag/tagmanifest-sha256.txt @@ -0,0 +1,3 @@ +d0d7117f5662fe588aab40c61c66163e961c930ffec5fd52d64d39a6aa8f8e38 bag-info.txt +e91f941be5973ff71f1dccbdd1a32d598881893a7f21be516aca743da38b1689 bagit.txt +ba57368e0165a727b0b6cc5f040c5cbd3e3ac28d1d52f94d111d64382c6f0466 manifest-sha256.txt diff --git a/src/test/resources/sha512Bag/bag-info.txt b/src/test/resources/sha512Bag/bag-info.txt new file mode 100644 index 000000000..ec9f2e4da --- /dev/null +++ b/src/test/resources/sha512Bag/bag-info.txt @@ -0,0 +1,3 @@ +Bag-Software-Agent: bagit.py v1.7.0 +Bagging-Date: 2018-07-24 +Payload-Oxum: 44.1 diff --git a/src/test/resources/sha512Bag/bagit.txt b/src/test/resources/sha512Bag/bagit.txt new file mode 100644 index 000000000..c4aebb43a --- /dev/null +++ b/src/test/resources/sha512Bag/bagit.txt @@ -0,0 +1,2 @@ +BagIt-Version: 0.97 +Tag-File-Character-Encoding: UTF-8 diff --git a/src/test/resources/sha512Bag/data/readme.txt b/src/test/resources/sha512Bag/data/readme.txt new file mode 100644 index 000000000..218b84225 --- /dev/null +++ b/src/test/resources/sha512Bag/data/readme.txt @@ -0,0 +1 @@ +this is a valid sha512 bag used for testing diff --git a/src/test/resources/sha512Bag/manifest-sha512.txt b/src/test/resources/sha512Bag/manifest-sha512.txt new file mode 100644 index 000000000..d6f3bc3dd --- /dev/null +++ b/src/test/resources/sha512Bag/manifest-sha512.txt @@ -0,0 +1 @@ +6e464149951ad3e3975c326ccb5c6e4ce306516fcc99689d90094e3bfbd87e87e0f7dbb4efd8e59456b808ca2b7a22bb14fa0b1edb76e11964a181850b45a628 data/readme.txt diff --git a/src/test/resources/sha512Bag/tagmanifest-sha512.txt b/src/test/resources/sha512Bag/tagmanifest-sha512.txt new file mode 100644 index 000000000..f9bf3e71a --- /dev/null +++ b/src/test/resources/sha512Bag/tagmanifest-sha512.txt @@ -0,0 +1,3 @@ +bede8e68c6fef4be00b54e5a5c2e104501444531679948187926a84a4e5c85d466959f34dae9ac4c81ccea4ec28e3f0ea7b661c0f100e457b3a568a08661cc01 bag-info.txt +418dcfbe17d5f4b454b18630be795462cf7da4ceb6313afa49451aa2568e41f7ca3d34cf0280c7d056dc5681a70c37586aa1755620520b9198eede905ba2d0f6 bagit.txt +82e79660ff0335b94f56a0b837b1564291b3eb2242a8c28c51d32efc6db9db5ab6fe7656d975bf16989c67ef3a3fa5cbd066eb1bcf422207ec41a20ed0f98943 manifest-sha512.txt From a515359c65e3ee133b3f9fc6f7cc0612f0241177 Mon Sep 17 00:00:00 2001 From: John Scancella Date: Tue, 24 Jul 2018 12:58:58 -0400 Subject: [PATCH 6/9] refs #119 - reverting change to name of message digests --- .../gov/loc/repository/bagit/BagitSuiteComplanceTest.java | 1 + .../bagit/hash/StandardSupportedAlgorithms.java | 8 ++++---- .../java/gov/loc/repository/bagit/TempFolderTest.java | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java b/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java index 24cd32220..eb8827dbe 100644 --- a/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java +++ b/src/integration/java/gov/loc/repository/bagit/BagitSuiteComplanceTest.java @@ -147,6 +147,7 @@ public void testReadWriteProducesSameBag() throws Exception{ testTagFileContents(bag, newBagDir); testBagsStructureAreEqual(bagDir, newBagDir); + delete(newBagDir); } } diff --git a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java index 32df24a7e..40818316c 100644 --- a/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java +++ b/src/main/java/gov/loc/repository/bagit/hash/StandardSupportedAlgorithms.java @@ -5,10 +5,10 @@ */ public enum StandardSupportedAlgorithms implements SupportedAlgorithm{ MD5("MD5"), - SHA1("SHA1"), - SHA224("SHA224"), - SHA256("SHA256"), - SHA512("SHA512"); + SHA1("SHA-1"), + SHA224("SHA-224"), + SHA256("SHA-256"), + SHA512("SHA-512"); private final String messageDigestName; diff --git a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java b/src/test/java/gov/loc/repository/bagit/TempFolderTest.java index 591345d2a..80b3a73c2 100644 --- a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java +++ b/src/test/java/gov/loc/repository/bagit/TempFolderTest.java @@ -8,6 +8,7 @@ import java.nio.file.attribute.BasicFileAttributes; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; abstract public class TempFolderTest { @@ -21,6 +22,8 @@ public void setupTempFolder() throws IOException{ @AfterEach public void teardownTempFolder() throws IOException{ delete(folder); + Assertions.assertFalse(Files.exists(folder)); + //Assertions.assertEquals(0, Files.list(folder).count()); } public Path createDirectory(String name) throws IOException { @@ -33,7 +36,7 @@ public Path createFile(String name) throws IOException { return Files.createFile(newFile); } - private void delete(Path tempDirectory) throws IOException { + protected void delete(Path tempDirectory) throws IOException { Files.walkFileTree(tempDirectory, new SimpleFileVisitor() { @Override From c782e26dcfdc51efce7d9b64375aa4d01bdd332e Mon Sep 17 00:00:00 2001 From: John Scancella Date: Tue, 24 Jul 2018 13:03:00 -0400 Subject: [PATCH 7/9] Fixed error of integration tests not being run after converting to junit 5 --- code-quality.gradle | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/code-quality.gradle b/code-quality.gradle index eee506764..e8c0a7026 100644 --- a/code-quality.gradle +++ b/code-quality.gradle @@ -22,7 +22,17 @@ task integrationTest(type: Test, dependsOn: "cloneConformanceSuite") { description "Runs the integration tests." testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath - testLogging.showStandardStreams = true + //testLogging.showStandardStreams = true + + testLogging { + events "passed", "skipped", "failed" + } + useJUnitPlatform() + + jacoco { + destinationFile = file("$buildDir/jacoco/integrationTest.exec") + classDumpDir = file("$buildDir/classes/integrationTest") + } } jacocoTestReport.dependsOn integrationTest //include the integration tests in the code coverage reports From c2e6025e1035d6dfbbbaaa5250218f8e0bbae5ec Mon Sep 17 00:00:00 2001 From: John Scancella Date: Tue, 24 Jul 2018 15:42:49 -0400 Subject: [PATCH 8/9] don't do code coverage on integration tests --- code-quality.gradle | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/code-quality.gradle b/code-quality.gradle index e8c0a7026..899b67114 100644 --- a/code-quality.gradle +++ b/code-quality.gradle @@ -23,16 +23,11 @@ task integrationTest(type: Test, dependsOn: "cloneConformanceSuite") { testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath //testLogging.showStandardStreams = true + useJUnitPlatform() testLogging { events "passed", "skipped", "failed" } - useJUnitPlatform() - - jacoco { - destinationFile = file("$buildDir/jacoco/integrationTest.exec") - classDumpDir = file("$buildDir/classes/integrationTest") - } } jacocoTestReport.dependsOn integrationTest //include the integration tests in the code coverage reports From a06001ce68b91b2f532a9ec23979f631621ce309 Mon Sep 17 00:00:00 2001 From: John Scancella Date: Fri, 27 Jul 2018 12:57:27 -0400 Subject: [PATCH 9/9] refs #122 - skip files that are hidden when checking if file is in at least one manifest when option is enabled --- .../repository/bagit/verify/BagVerifier.java | 4 ++- ...dFileExistsInAtLeastOneManifestVistor.java | 17 +++++++--- src/main/resources/MessageBundle.properties | 3 ++ .../loc/repository/bagit/TempFolderTest.java | 18 ++++++++++ .../gov/loc/repository/bagit/TestUtils.java | 6 ++++ .../bagit/verify/BagVerifierTest.java | 33 +++++++++++++++++++ 6 files changed, 76 insertions(+), 5 deletions(-) diff --git a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java index 663cd8b8f..8e5e831bb 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java +++ b/src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java @@ -17,6 +17,7 @@ import gov.loc.repository.bagit.domain.Bag; import gov.loc.repository.bagit.domain.Manifest; import gov.loc.repository.bagit.exceptions.CorruptChecksumException; +import gov.loc.repository.bagit.exceptions.FileNotInManifestException; import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException; import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException; import gov.loc.repository.bagit.exceptions.InvalidPayloadOxumException; @@ -120,6 +121,7 @@ public static void quicklyVerify(final Bag bag) throws IOException, InvalidPaylo * * @throws CorruptChecksumException when the computed hash doesn't match given hash * @throws IOException if there was an error with the file + * @throws FileNotInManifestException if a file is found in the payload directory but not in manifest(s) * @throws MissingPayloadManifestException if there is not at least one payload manifest * @throws MissingBagitFileException if there is no bagit.txt file * @throws MissingPayloadDirectoryException if there is no /data directory @@ -130,7 +132,7 @@ public static void quicklyVerify(final Bag bag) throws IOException, InvalidPaylo * @throws UnsupportedAlgorithmException if the manifest uses a algorithm that isn't supported * @throws InvalidBagitFileFormatException if the manifest is not formatted properly */ - public void isValid(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ + public void isValid(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, FileNotInManifestException, MissingPayloadManifestException, MissingBagitFileException, MissingPayloadDirectoryException, FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, CorruptChecksumException, VerificationException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{ logger.info(messages.getString("checking_bag_is_valid"), bag.getRootDir()); isComplete(bag, ignoreHiddenFiles); diff --git a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java b/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java index 0bf8cd0e3..ea6077f6c 100644 --- a/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java +++ b/src/main/java/gov/loc/repository/bagit/verify/PayloadFileExistsInAtLeastOneManifestVistor.java @@ -1,10 +1,12 @@ package gov.loc.repository.bagit.verify; +import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ResourceBundle; import java.util.Set; import org.slf4j.helpers.MessageFormatter; @@ -15,6 +17,7 @@ * Implements {@link SimpleFileVisitor} to ensure that the encountered file is in one of the manifests. */ public class PayloadFileExistsInAtLeastOneManifestVistor extends AbstractPayloadFileExistsInManifestsVistor { + private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle"); private transient final Set filesListedInManifests; public PayloadFileExistsInAtLeastOneManifestVistor(final Set filesListedInManifests, final boolean ignoreHiddenFiles) { @@ -23,12 +26,18 @@ public PayloadFileExistsInAtLeastOneManifestVistor(final Set filesListedIn } @Override - public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws FileNotInManifestException{ - if(Files.isRegularFile(path) && !filesListedInManifests.contains(path.normalize())){ + public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs)throws IOException, FileNotInManifestException{ + if(Files.isHidden(path) && ignoreHiddenFiles){ + logger.debug(messages.getString("skipping_hidden_file"), path); + } + else { + if(Files.isRegularFile(path) && !filesListedInManifests.contains(path.normalize())){ final String formattedMessage = messages.getString("file_not_in_any_manifest_error"); throw new FileNotInManifestException(MessageFormatter.format(formattedMessage, path).getMessage()); } - logger.debug("[{}] is in at least one manifest", path); - return FileVisitResult.CONTINUE; + logger.debug(messages.getString("file_in_at_least_one_manifest"), path); + } + return FileVisitResult.CONTINUE; } + } diff --git a/src/main/resources/MessageBundle.properties b/src/main/resources/MessageBundle.properties index 4973bd2e8..0c69d7953 100644 --- a/src/main/resources/MessageBundle.properties +++ b/src/main/resources/MessageBundle.properties @@ -182,6 +182,9 @@ file_not_in_manifest_error=File [{}] is in the payload directory but isn't liste file_in_all_manifests=[{}] is in all manifests. file_not_in_any_manifest_error=File [{}] is in the payload directory but isn't listed in any manifest! +#for PayloadFileExistsInAtLeastOneManifestVistor.java +file_in_at_least_one_manifest="[{}] is in at least one manifest" + #for PayloadVerifier.java all_files_in_manifests=Getting all files listed in the manifest(s). get_listing_in_manifest=Getting files and checksums listed in [{}]. diff --git a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java b/src/test/java/gov/loc/repository/bagit/TempFolderTest.java index 80b3a73c2..471c21039 100644 --- a/src/test/java/gov/loc/repository/bagit/TempFolderTest.java +++ b/src/test/java/gov/loc/repository/bagit/TempFolderTest.java @@ -36,6 +36,24 @@ public Path createFile(String name) throws IOException { return Files.createFile(newFile); } + public Path copyBagToTempFolder(Path bagFolder) throws IOException{ + Path bagCopyDir = createDirectory(bagFolder.getFileName() + "_copy"); + Files.walkFileTree(bagFolder, new SimpleFileVisitor() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path relative = bagFolder.relativize(file); + if(relative.getParent() != null) { + Files.createDirectories(bagCopyDir.resolve(relative.getParent())); + } + Files.copy(file, bagCopyDir.resolve(relative)); + return FileVisitResult.CONTINUE; + } + }); + + return bagCopyDir; + } + protected void delete(Path tempDirectory) throws IOException { Files.walkFileTree(tempDirectory, new SimpleFileVisitor() { diff --git a/src/test/java/gov/loc/repository/bagit/TestUtils.java b/src/test/java/gov/loc/repository/bagit/TestUtils.java index 619760301..3c29083ad 100644 --- a/src/test/java/gov/loc/repository/bagit/TestUtils.java +++ b/src/test/java/gov/loc/repository/bagit/TestUtils.java @@ -12,6 +12,12 @@ public static boolean isExecutingOnWindows(){ return System.getProperty("os.name").contains("Windows"); } + /** + * walk a directory and make sure that files/folders are hidden if they start with a . on windows. + * + * @param startingDir the directory to start walking + * @throws IOException if there is a problem setting the file/folder to be hidden + */ public static void makeFilesHiddenOnWindows(Path startingDir) throws IOException { if (isExecutingOnWindows()) { Files.walkFileTree(startingDir, new SimpleFileVisitor() { diff --git a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java b/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java index 68955a149..8099091b4 100644 --- a/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java +++ b/src/test/java/gov/loc/repository/bagit/verify/BagVerifierTest.java @@ -1,6 +1,7 @@ package gov.loc.repository.bagit.verify; import java.io.File; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.Security; @@ -12,9 +13,11 @@ import org.junit.jupiter.api.Test; import gov.loc.repository.bagit.TempFolderTest; +import gov.loc.repository.bagit.TestUtils; import gov.loc.repository.bagit.domain.Bag; import gov.loc.repository.bagit.domain.Manifest; import gov.loc.repository.bagit.exceptions.CorruptChecksumException; +import gov.loc.repository.bagit.exceptions.FileNotInManifestException; import gov.loc.repository.bagit.exceptions.UnsupportedAlgorithmException; import gov.loc.repository.bagit.exceptions.VerificationException; import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms; @@ -33,6 +36,36 @@ public class BagVerifierTest extends TempFolderTest{ private BagVerifier sut = new BagVerifier(); private BagReader reader = new BagReader(); + @Test + public void testValidWhenHiddenFolderNotIncluded() throws Exception{ + Path copyDir = copyBagToTempFolder(rootDir); + Files.createDirectory(copyDir.resolve("data").resolve(".someHiddenFolder")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + sut.isValid(bag, true); + } + + @Test + public void testValidWithHiddenFile() throws Exception{ + Path copyDir = copyBagToTempFolder(rootDir); + Files.createFile(copyDir.resolve("data").resolve(".someHiddenFile")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + sut.isValid(bag, true); + } + + @Test + public void testInvalidWithHiddenFile() throws Exception{ + Path copyDir = copyBagToTempFolder(rootDir); + Files.createFile(copyDir.resolve("data").resolve(".someHiddenFile")); + TestUtils.makeFilesHiddenOnWindows(copyDir); + + Bag bag = reader.read(copyDir); + Assertions.assertThrows(FileNotInManifestException.class, () -> { sut.isValid(bag, false); }); + } + @Test public void testStandardSupportedAlgorithms() throws Exception{ List algorithms = Arrays.asList("md5", "sha1", "sha256", "sha512");