Write md5sums for extracted files

NitorCreations · Nov 5, 2015 · b3294b0 · b3294b0
1 parent f732e21
commit b3294b0
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 21 deletions.
diff --git a/willow-download/src/main/java/com/nitorcreations/willow/download/Extractor.java b/willow-download/src/main/java/com/nitorcreations/willow/download/Extractor.java
@@ -5,15 +5,19 @@
 import static com.nitorcreations.willow.properties.PropertyKeys.PROPERTY_KEY_SUFFIX_EXTRACT_OVERWRITE;
 import static com.nitorcreations.willow.properties.PropertyKeys.PROPERTY_KEY_SUFFIX_EXTRACT_ROOT;
 import static com.nitorcreations.willow.properties.PropertyKeys.PROPERTY_KEY_SUFFIX_EXTRACT_SKIP_GLOB;
+import static com.nitorcreations.willow.properties.PropertyKeys.PROPERTY_KEY_SUFFIX_WRITE_MD5SUMS;
 import static com.nitorcreations.willow.properties.PropertyKeys.PROPERTY_KEY_WORKDIR;
 
 import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -34,6 +38,8 @@
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
+import javax.xml.bind.DatatypeConverter;
+
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveException;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
@@ -44,14 +50,15 @@
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
 
 import com.nitorcreations.willow.utils.FileUtil;
+import com.nitorcreations.willow.utils.MD5SumOutputStream;
 
 @SuppressWarnings({"PMD.TooManyStaticImports", "PMD.AvoidUsingOctalValues"})
 public class Extractor implements Callable<Boolean> {
   private final Properties properties;
   private final File target;
   private final Logger logger;
   private static final Pattern zipFilesPattern = Pattern.compile(".*?\\.zip$|.*?\\.jar$|.*?\\.war$|.*?\\.ear$", Pattern.CASE_INSENSITIVE);
-  private static final Pattern compressedPattern = Pattern.compile(".*?\\.gz$|.*?\\.tgz$|.*?\\.z$|.*?\\.bz2$|.*?\\.lzma$|.*?\\.arj$|.*?\\.deflate$");
+  private static final Pattern compressedPattern = Pattern.compile(".*?\\.gz$|.*?\\.tgz$|.*?\\.z$|.*?\\.bz2$|.*?\\.lzma$|.*?\\.arj$|.*?\\.deflate$", Pattern.CASE_INSENSITIVE);
   private final ArchiveStreamFactory factory = new ArchiveStreamFactory();
   private final CompressorStreamFactory cfactory = new CompressorStreamFactory();
   private static Map<Integer, PosixFilePermission> perms = new HashMap<Integer, PosixFilePermission>();
@@ -84,6 +91,7 @@ public Boolean call() {
     String extractGlob = properties.getProperty(PROPERTY_KEY_SUFFIX_EXTRACT_GLOB);
     String skipExtractGlob = properties.getProperty(PROPERTY_KEY_SUFFIX_EXTRACT_SKIP_GLOB);
     String filterGlob = properties.getProperty(PROPERTY_KEY_SUFFIX_EXTRACT_FILTER_GLOB);
+    boolean writeMd5Sums = "true".equalsIgnoreCase(properties.getProperty(PROPERTY_KEY_SUFFIX_WRITE_MD5SUMS, "false"));
     File root = new File(extractRoot);
     Map<String, String> replaceParameters = new HashMap<>();
     for (Entry<Object, Object> next : properties.entrySet()) {
@@ -95,7 +103,7 @@ public Boolean call() {
       }
       int entries = 0;
       if (extractGlob != null || skipExtractGlob != null) {
-        entries = extractFile(target, replaceParameters, root, extractGlob, skipExtractGlob, filterGlob, overwrite);
+        entries = extractFile(target, replaceParameters, root, extractGlob, skipExtractGlob, filterGlob, overwrite, writeMd5Sums);
       }
       logger.log(Level.INFO, "Processed " + entries + " entries");
       return true;
@@ -105,47 +113,47 @@ public Boolean call() {
     }
   }
 
-  private int extractFile(File archive, Map<String, String> replaceTokens, File root, String extractGlob, String skipExtractGlob, String filterGlob, boolean overwrite) throws CompressorException, IOException, ArchiveException {
+  private int extractFile(File archive, Map<String, String> replaceTokens, File root, String extractGlob, String skipExtractGlob, String filterGlob, boolean overwrite, boolean writeMd5Sums) throws CompressorException, IOException, ArchiveException {
     Set<PathMatcher> extractMatchers = getGlobMatchers(extractGlob);
     Set<PathMatcher> skipMatchers = getGlobMatchers(skipExtractGlob);
     Set<PathMatcher> filterMatchers = getGlobMatchers(filterGlob);
     if (zipFilesPattern.matcher(archive.getName()).matches()) {
       try (ZipFile source = new ZipFile(archive)) {
-        return extractZip(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite);
+        return extractZip(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite, writeMd5Sums);
       }
     } else {
       try (InputStream in = new BufferedInputStream(new FileInputStream(archive), 8 * 1024)) {
         if (compressedPattern.matcher(archive.getName()).matches()) {
           try (InputStream compressed = new BufferedInputStream(cfactory.createCompressorInputStream(in), 8 * 1024)) {
             try (ArchiveInputStream source = factory.createArchiveInputStream(compressed)) {
-              return extractArchive(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite);
+              return extractArchive(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite, writeMd5Sums);
             }
           }
         } else {
           try (ArchiveInputStream source = factory.createArchiveInputStream(in)) {
-            return extractArchive(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite);
+            return extractArchive(source, root, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite, writeMd5Sums);
           }
         }
       }
     }
   }
 
-  private int extractArchive(ArchiveInputStream is, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite) throws IOException {
+  private int extractArchive(ArchiveInputStream is, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite, boolean writeMd5Sums) throws IOException {
     ArchiveEntry entry;
     int entries = 0;
     while ((entry = is.getNextEntry()) != null) {
-      extractEntry(is, entry, destFolder, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite);
+      extractEntry(is, entry, destFolder, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite, writeMd5Sums);
       entries++;
     }
     return entries;
   }
 
-  private int extractZip(ZipFile zipFile, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite) throws IOException {
+  private int extractZip(ZipFile zipFile, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite, boolean writeMd5Sums) throws IOException {
     Enumeration<ZipArchiveEntry> en = zipFile.getEntries();
     int entries = 0;
     while (en.hasMoreElements()) {
       ZipArchiveEntry nextEntry = en.nextElement();
-      extractEntry(zipFile.getInputStream(nextEntry), nextEntry, destFolder, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite);
+      extractEntry(zipFile.getInputStream(nextEntry), nextEntry, destFolder, replaceTokens, extractMatchers, skipMatchers, filterMatchers, overwrite, writeMd5Sums);
       entries++;
     }
     return entries;
@@ -186,7 +194,7 @@ public Set<PosixFilePermission> getPermissions(int mode) {
     return permissions;
   }
   @SuppressWarnings("PMD.CollapsibleIfStatements")
-  private void extractEntry(InputStream is, ArchiveEntry entry, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite) throws IOException {
+  private void extractEntry(InputStream is, ArchiveEntry entry, File destFolder, Map<String, String> replaceTokens, Set<PathMatcher> extractMatchers, Set<PathMatcher> skipMatchers, Set<PathMatcher> filterMatchers, boolean overwrite, boolean writeMd5Sums) throws IOException {
     File dest = new File(destFolder, entry.getName()).getCanonicalFile();
     if (globMatches(entry.getName(), extractMatchers) && !globMatches(entry.getName(), skipMatchers)) {
       if (entry.isDirectory()) {
@@ -196,10 +204,22 @@ private void extractEntry(InputStream is, ArchiveEntry entry, File destFolder, M
           return;
         }
         FileUtil.createDir(dest.getParentFile());
-        if (globMatches(entry.getName(), filterMatchers)) {
-          FileUtil.filterStream(is, dest, replaceTokens);
-        } else {
-          FileUtil.copy(is, dest);
+        try (OutputStream out = writeMd5Sums ? new MD5SumOutputStream(new FileOutputStream(dest)) : new FileOutputStream(dest)) {
+          if (globMatches(entry.getName(), filterMatchers)) {
+            FileUtil.filterStream(is, out, replaceTokens);
+          } else {
+            FileUtil.copy(is, out);
+          }
+          if (out instanceof MD5SumOutputStream) {
+            byte[] digest = ((MD5SumOutputStream)out).digest();
+            File md5file = new File(dest.getParentFile(), dest.getName() + ".md5");
+            String md5str = DatatypeConverter.printHexBinary(digest);
+            try (OutputStream mdout = new FileOutputStream(md5file)) {
+              mdout.write((md5str + "  " + dest.getName() + "\n").getBytes(StandardCharsets.UTF_8));
+              mdout.flush();
+            }
+
+          }
         }
         PosixFileAttributeView posix = Files.getFileAttributeView(dest.toPath(), PosixFileAttributeView.class);
         Set<PosixFilePermission> permissions = getPermissions(getMode(entry));

diff --git a/willow-download/src/main/java/com/nitorcreations/willow/download/UrlDownloader.java b/willow-download/src/main/java/com/nitorcreations/willow/download/UrlDownloader.java
@@ -20,7 +20,6 @@
 import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
-import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 import java.util.Properties;
 import java.util.concurrent.Callable;
@@ -145,7 +144,7 @@ public void log(long downloaded) {
           }
         }
         break;
-      } catch (URISyntaxException | IOException | NoSuchAlgorithmException e) {
+      } catch (URISyntaxException | IOException e) {
         tryNo++;
         target = null;
         logger.log(Level.WARNING, "Failed to download and extract " + url, e);

diff --git a/willow-servers/src/main/java/com/nitorcreations/willow/servers/DeploymentServlet.java b/willow-servers/src/main/java/com/nitorcreations/willow/servers/DeploymentServlet.java
@@ -179,6 +179,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
     Properties extAll = new Properties();
     extAll.setProperty(PropertyKeys.PROPERTY_KEY_SUFFIX_EXTRACT_GLOB, "**");
     extAll.setProperty(PropertyKeys.PROPERTY_KEY_SUFFIX_EXTRACT_ROOT, versionRoot.getAbsolutePath());
+    extAll.setProperty(PropertyKeys.PROPERTY_KEY_SUFFIX_WRITE_MD5SUMS, "true");
+
     Extractor extract = new Extractor(extAll, target);
     extract.call();
     File props = new File(versionRoot, "properties.jar");

diff --git a/willow-utils/src/main/java/com/nitorcreations/willow/properties/PropertyKeys.java b/willow-utils/src/main/java/com/nitorcreations/willow/properties/PropertyKeys.java
@@ -48,6 +48,7 @@ public interface PropertyKeys {
   String PROPERTY_KEY_SUFFIX_EXTRACT_GLOB = "extract.glob";
   String PROPERTY_KEY_SUFFIX_EXTRACT_SKIP_GLOB = "extract.skip.glob";
   String PROPERTY_KEY_SUFFIX_EXTRACT_FILTER_GLOB = "extract.filter.glob";
+  String PROPERTY_KEY_SUFFIX_WRITE_MD5SUMS = "extract.writemd5sums";
   String PROPERTY_KEY_SUFFIX_EXTRACT_OVERWRITE = "extract.overwrite";
   String PROPERTY_KEY_SUFFIX_PROXYAUTOCONF = "proxyautoconf";
   String PROPERTY_KEY_SUFFIX_PROXY = "proxy";

diff --git a/willow-utils/src/main/java/com/nitorcreations/willow/utils/FileUtil.java b/willow-utils/src/main/java/com/nitorcreations/willow/utils/FileUtil.java
@@ -88,10 +88,10 @@ public static long filterStream(InputStream original, File target, Map<String, S
     }
   }
 
-  public static long filterStream(InputStream original, FileOutputStream out, Map<String, String> replaceTokens) throws IOException {
+  public static long filterStream(InputStream original, OutputStream out, Map<String, String> replaceTokens) throws IOException {
     return filterStream(original, out, replaceTokens, null);
   }
-  public static long filterStream(InputStream original, FileOutputStream out, Map<String, String> replaceTokens, DownloadLogger logger) throws IOException {
+  public static long filterStream(InputStream original, OutputStream out, Map<String, String> replaceTokens, DownloadLogger logger) throws IOException {
     try (InputStream in = new ReplaceTokensInputStream(new BufferedInputStream(original, BUFFER_LEN), StandardCharsets.UTF_8, replaceTokens, AT_DELIMITERS, CURLY_DELIMITERS)) {
       long length = copyByteByByte(in, out, logger);
       return length;

diff --git a/willow-utils/src/main/java/com/nitorcreations/willow/utils/MD5SumInputStream.java b/willow-utils/src/main/java/com/nitorcreations/willow/utils/MD5SumInputStream.java
@@ -12,8 +12,15 @@
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
 public class MD5SumInputStream extends DigestInputStream {
-  public MD5SumInputStream(final InputStream stream) throws NoSuchAlgorithmException {
-    super(stream, MessageDigest.getInstance("MD5"));
+  public MD5SumInputStream(final InputStream stream) {
+    super(stream, getMD5Digest());
+  }
+  public static MessageDigest getMD5Digest() {
+    try {
+      return MessageDigest.getInstance("MD5");
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException("MD5 message digest not available", e);
+    }
   }
   public byte[] digest() {
     return getMessageDigest().digest();

diff --git a/willow-utils/src/main/java/com/nitorcreations/willow/utils/MD5SumOutputStream.java b/willow-utils/src/main/java/com/nitorcreations/willow/utils/MD5SumOutputStream.java
@@ -0,0 +1,15 @@
+package com.nitorcreations.willow.utils;
+
+import java.io.OutputStream;
+import java.security.DigestOutputStream;
+
+public class MD5SumOutputStream extends DigestOutputStream {
+
+  public MD5SumOutputStream(OutputStream stream) {
+    super(stream, MD5SumInputStream.getMD5Digest());
+  }
+  public byte[] digest() {
+    return getMessageDigest().digest();
+  }
+
+}