We now use the FastMD5 implementation for MD5 checksums.

NOTICE

@@ -1,5 +1,5 @@
-The product includes software from the ehcache3 project,
+The product includes software from the ehcache3 project
 under the Apache License 2.0 (see: com.joyent.manta.util.ConcurrentWeakIdentityHashMap).
 
-The product includes software from the Apache HTTPClient project,
+The product includes software from the Apache HTTPClient project
 under the Apache License 2.0 (see: com.joyent.manta.util.NotThreadSafe).

README.md

@@ -22,9 +22,9 @@ SDK for interacting with Joyent's Manta system.
 #### CLI Requirements
 
 Add [BouncyCastle](http://www.bouncycastle.org/latest_releases.html) as a security provider
- 1. Edit "jre\lib\security\java.security
" Add an entry for BouncyCastle  
+ 1. Edit "$JAVA_HOME/jre/lib/security/java.security
" Add an entry for BouncyCastle  
  `security.provider.11=org.bouncycastle.jce.provider.BouncyCastleProvider`
- 2. Copy bc*.jar to jre\lib\ext
+ 2. Copy bc*.jar to $JAVA_HOME/jre/lib/ext
 
 ### Using Maven
 Add the latest java-manta dependency to your Maven `pom.xml`.

java-manta-benchmark/src/main/java/com/joyent/manta/benchmark/EncryptingEntityBenchmark.java

@@ -13,6 +13,7 @@
 import com.joyent.manta.client.crypto.SupportedCiphersLookupMap;
 import com.joyent.manta.http.entity.DigestedEntity;
 import com.joyent.manta.http.entity.MantaInputStreamEntity;
+import com.twmacinta.util.FastMD5Digest;
 import org.apache.commons.io.output.NullOutputStream;
 
 import javax.crypto.SecretKey;
@@ -60,7 +61,7 @@ private static void throughputTest(final int tries,
                         cipherDetails, entity);
 
                 DigestedEntity digestedEntity = new DigestedEntity(encryptingEntity,
-                        "MD5");
+                        new FastMD5Digest());
 
                 long start = System.nanoTime();
                 digestedEntity.writeTo(noopOut);

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/EncryptionStateSerializer.java

@@ -76,7 +76,7 @@ private void registerClasses(final Kryo kryo) {
         kryo.register(AesCbcCipherDetails.class, new SupportedCipherDetailsSerializer());
         kryo.register(AesGcmCipherDetails.class, new SupportedCipherDetailsSerializer());
         kryo.register(Cipher.class, new CipherSerializer(kryo));
-        kryo.register(HMac.class, new HmacSerializer());
+        kryo.register(HMac.class, new HmacSerializer(kryo));
 
         final Class<?> closeShieldStreamClass = findCloseShieldStreamClass();
         Objects.requireNonNull(closeShieldStreamClass,

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/HmacSerializer.java

@@ -10,9 +10,17 @@
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
+import com.esotericsoftware.kryo.serializers.CompatibleFieldSerializer;
+import com.twmacinta.util.FastMD5Digest;
+import com.twmacinta.util.MD5;
 import org.bouncycastle.crypto.Digest;
 import org.bouncycastle.crypto.digests.EncodableDigest;
 import org.bouncycastle.crypto.digests.GeneralDigest;
+import org.bouncycastle.crypto.digests.MD5Digest;
+import org.bouncycastle.crypto.digests.SHA1Digest;
+import org.bouncycastle.crypto.digests.SHA256Digest;
+import org.bouncycastle.crypto.digests.SHA384Digest;
+import org.bouncycastle.crypto.digests.SHA512Digest;
 import org.bouncycastle.crypto.macs.HMac;
 
 import java.lang.reflect.Field;
@@ -40,9 +48,29 @@ public class HmacSerializer extends AbstractManualSerializer<HMac> {
 
     /**
      * Creates a new Kryo serializer for {@link HMac} objects.
+     * @param kryo Kryo serializer instance
      */
-    public HmacSerializer() {
+    public HmacSerializer(final Kryo kryo) {
         super(HMac.class, false);
+        registerClasses(kryo);
+    }
+
+    /**
+     * Registers the classes needed for serialization with Kryo.
+     *
+     * @param kryo Kryo instance
+     */
+    private void registerClasses(final Kryo kryo) {
+        kryo.register(HMac.class);
+        kryo.register(FastMD5Digest.class);
+        kryo.register(MD5.class);
+        Class<?> md5StateClass = ReflectionUtils.findClass("com.twmacinta.util.MD5State");
+        kryo.register(md5StateClass, new CompatibleFieldSerializer(kryo, md5StateClass));
+        kryo.register(MD5Digest.class);
+        kryo.register(SHA1Digest.class);
+        kryo.register(SHA256Digest.class);
+        kryo.register(SHA384Digest.class);
+        kryo.register(SHA512Digest.class);
     }
 
     @Override
@@ -83,4 +111,39 @@ public HMac read(final Kryo kryo, final Input input, final Class<HMac> type) {
 
         return hmac;
     }
+
+//    @Override
+//    @SuppressWarnings("unchecked")
+//    public void write(final Kryo kryo, final Output output, final HMac object) {
+//        final Digest digest = object.getUnderlyingDigest();
+//        kryo.writeClassAndObject(output, digest);
+//
+//        final EncodableDigest ipadState = (EncodableDigest)readField(ipadStateField, object);
+//        final EncodableDigest opadState = (EncodableDigest)readField(opadStateField, object);
+//
+//        output.writeInt(ipadState.getEncodedState().length);
+//        output.write(ipadState.getEncodedState());
+//        output.writeInt(opadState.getEncodedState().length);
+//        output.write(opadState.getEncodedState());
+//
+//        output.flush();
+//    }
+//
+//    @Override
+//    @SuppressWarnings("unchecked")
+//    public HMac read(final Kryo kryo, final Input input, final Class<HMac> type) {
+//        final Digest digest = (Digest)kryo.readClassAndObject(input);
+//
+//        byte[] ipadStateBytes = input.readBytes(input.readInt());
+//        Digest ipadState = ReflectionUtils.newInstance(digest.getClass(), new Object[] {ipadStateBytes});
+//        byte[] opadStateBytes = input.readBytes(input.readInt());
+//        Digest opadState = ReflectionUtils.newInstance(digest.getClass(), new Object[] {opadStateBytes});
+//
+//        HMac hmac = new HMac(digest);
+//
+//        writeField(ipadStateField, hmac, ipadState);
+//        writeField(opadStateField, hmac, opadState);
+//
+//        return hmac;
+//    }
 }

java-manta-client-kryo-serialization/src/test/java/com/joyent/manta/serialization/HmacSerializerTest.java

@@ -39,7 +39,7 @@ public class HmacSerializerTest {
 
     @BeforeClass
     public void setup() {
-        kryo.register(HMac.class, new HmacSerializer());
+        kryo.register(HMac.class, new HmacSerializer(kryo));
         SupportedCipherDetails cipherDetails = DefaultsConfigContext.DEFAULT_CIPHER;
         byte[] keyBytes = Base64.getDecoder().decode("qAnCNUmmFjUTtImNGv241Q==");
         this.secretKey = SecretKeyUtils.loadKey(keyBytes, cipherDetails);
@@ -90,6 +90,8 @@ private void canSerializeHmac(final String algorithm) throws Exception {
         try (ByteArrayOutputStream out = new ByteArrayOutputStream();
              Output output = new Output(out)) {
             kryo.writeObject(output, hmac);
+            output.flush();
+
             serializedContent = out.toByteArray();
         }
 

java-manta-client/pom.xml

@@ -142,6 +142,13 @@
             </exclusions>
         </dependency>
 
+        <!-- High speed MD5 implementation -->
+        <dependency>
+            <groupId>com.twmacinta</groupId>
+            <artifactId>fast-md5</artifactId>
+            <version>${dependency.fast-md5.version}</version>
+        </dependency>
+
         <!-- Logging bridge that will be shaded to bridge between the Apache
              Commons Logging framework and SLF4J -->
         <dependency>
@@ -201,6 +208,7 @@
                             <exclude>net.java.dev.jna:*</exclude>
                             <exclude>org.hamcrest:*</exclude>
                             <exclude>org.mockito:*</exclude>
+                            <exclude>com.twmacinta:*</exclude>
                         </excludes>
                     </artifactSet>
                     <relocations>

java-manta-client/src/main/java/com/joyent/manta/client/crypto/SupportedHmacsLookupMap.java

@@ -8,11 +8,11 @@
 package com.joyent.manta.client.crypto;
 
 import com.joyent.manta.exception.MantaClientEncryptionException;
+import com.twmacinta.util.FastMD5Digest;
 import com.joyent.manta.util.LookupMap;
 import com.joyent.manta.util.MantaUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
-import org.bouncycastle.crypto.digests.MD5Digest;
 import org.bouncycastle.crypto.digests.SHA1Digest;
 import org.bouncycastle.crypto.digests.SHA256Digest;
 import org.bouncycastle.crypto.digests.SHA384Digest;
@@ -59,7 +59,7 @@ private SupportedHmacsLookupMap() {
     private static Supplier<HMac> hmacSupplierByName(final String algorithm) {
         return () -> {
             if (algorithm.equalsIgnoreCase("HmacMD5")) {
-                return new HMac(new MD5Digest());
+                return new HMac(new FastMD5Digest());
             } else if (algorithm.equalsIgnoreCase("HmacSHA1")) {
                 return new HMac(new SHA1Digest());
             } else if (algorithm.equalsIgnoreCase("HmacSHA256")) {

java-manta-client/src/main/java/com/joyent/manta/http/StandardHttpHelper.java

@@ -19,7 +19,7 @@
 import com.joyent.manta.http.entity.DigestedEntity;
 import com.joyent.manta.http.entity.NoContentEntity;
 import com.joyent.manta.util.MantaUtils;
-import org.apache.commons.codec.digest.MessageDigestAlgorithms;
+import com.twmacinta.util.FastMD5Digest;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.Validate;
@@ -181,7 +181,7 @@ public MantaObjectResponse httpPut(final String path,
 
         if (entity != null) {
             if (validateUploadsEnabled()) {
-                md5DigestedEntity = new DigestedEntity(entity, MessageDigestAlgorithms.MD5);
+                md5DigestedEntity = new DigestedEntity(entity, new FastMD5Digest());
                 put.setEntity(md5DigestedEntity);
             } else {
                 md5DigestedEntity = null;
@@ -191,7 +191,8 @@ public MantaObjectResponse httpPut(final String path,
             md5DigestedEntity = null;
         }
 
-        CloseableHttpClient client = connectionContext.getHttpClient();
+        final CloseableHttpClient client = connectionContext.getHttpClient();
+        final MantaObjectResponse obj;
 
         try (CloseableHttpResponse response = client.execute(put)) {
             StatusLine statusLine = response.getStatusLine();
@@ -201,27 +202,27 @@ public MantaObjectResponse httpPut(final String path,
             // We add back in the metadata made in the request so that it is easily available
             responseHeaders.putAll(httpHeaders.metadata());
 
-            MantaObjectResponse obj = new MantaObjectResponse(path, responseHeaders, metadata);
+            obj = new MantaObjectResponse(path, responseHeaders, metadata);
 
             if (statusLine.getStatusCode() != HttpStatus.SC_NO_CONTENT) {
                 throw new MantaClientHttpResponseException(put, response,
                         put.getURI().getPath());
             }
+        }
 
-            /* We set the content type on the result object from the entity
-             * PUT if that content type isn't already present on the result object.
-             * This allows for the result object to have the original
-             * content-type even if it isn't part of any response headers. */
-            if (obj.getContentType() == null && entity != null && entity.getContentType() != null) {
-                obj.setContentType(entity.getContentType().getValue());
-            }
-
-            if (validateUploadsEnabled()) {
-                validateChecksum(md5DigestedEntity, obj.getMd5Bytes());
-            }
+        /* We set the content type on the result object from the entity
+         * PUT if that content type isn't already present on the result object.
+         * This allows for the result object to have the original
+         * content-type even if it isn't part of any response headers. */
+        if (obj.getContentType() == null && entity != null && entity.getContentType() != null) {
+            obj.setContentType(entity.getContentType().getValue());
+        }
 
-            return obj;
+        if (validateUploadsEnabled()) {
+            validateChecksum(md5DigestedEntity, obj.getMd5Bytes());
         }
+
+        return obj;
     }
 
     @Override

java-manta-client/src/main/java/com/joyent/manta/http/entity/DigestedEntity.java

@@ -7,18 +7,18 @@
  */
 package com.joyent.manta.http.entity;
 
-import com.joyent.manta.exception.MantaException;
 import com.joyent.manta.util.MantaUtils;
+import org.apache.commons.io.output.TeeOutputStream;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.bouncycastle.crypto.Digest;
+import org.bouncycastle.crypto.io.DigestOutputStream;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
+import java.nio.ByteBuffer;
 
 /**
  * Class that wraps an {@link HttpEntity} instance and calculates a running
@@ -31,7 +31,7 @@ public class DigestedEntity implements HttpEntity {
     /**
      * Calculates a running MD5 as data is streamed out.
      */
-    private final MessageDigest messageDigest;
+    private final Digest digest;
 
     /**
      * Wrapped entity implementation in which the API is proxied through.
@@ -43,19 +43,12 @@ public class DigestedEntity implements HttpEntity {
      * digest.
      *
      * @param wrapped entity to wrap
-     * @param algorithm lookup name of cryptographic digest
+     * @param digest cryptographic digest implementation
      */
     public DigestedEntity(final HttpEntity wrapped,
-                   final String algorithm) {
+                          final Digest digest) {
         this.wrapped = wrapped;
-
-        try {
-            this.messageDigest = MessageDigest.getInstance(algorithm);
-        } catch (NoSuchAlgorithmException e) {
-            String msg = String.format("No digest algorithm by the name of "
-                    + "[%s] available in the JVM", algorithm);
-            throw new MantaException(msg);
-        }
+        this.digest = digest;
     }
 
     @Override
@@ -93,13 +86,25 @@ public void writeTo(final OutputStream out) throws IOException {
         // If our wrapped entity is backed by a buffer of some form
         // we can read easily read the whole buffer into our message digest.
         if (wrapped instanceof MemoryBackedEntity) {
-            MemoryBackedEntity entity = (MemoryBackedEntity)wrapped;
-            messageDigest.update(entity.getBackingBuffer());
+            final MemoryBackedEntity entity = (MemoryBackedEntity)wrapped;
+            final ByteBuffer backingBuffer = entity.getBackingBuffer();
+
+            if (backingBuffer.hasArray()) {
+                final byte[] bytes = backingBuffer.array();
+                final int offset = backingBuffer.arrayOffset();
+                final int position = backingBuffer.position();
+                final int limit = backingBuffer.limit();
+
+                digest.update(bytes, offset + position, limit - position);
+                backingBuffer.position(limit);
 
-            wrapped.writeTo(out);
+                wrapped.writeTo(out);
+            }
         } else {
-            try (DigestOutputStream dout = new DigestOutputStream(out, messageDigest)) {
-                wrapped.writeTo(dout);
+            try (DigestOutputStream dout = new DigestOutputStream(digest);
+                 TeeOutputStream teeOut = new TeeOutputStream(out, dout)) {
+                wrapped.writeTo(teeOut);
+                teeOut.flush();
             }
         }
     }
@@ -122,13 +127,17 @@ public void consumeContent() throws IOException {
      * @return a byte array containing the md5 value
      */
     public byte[] getDigest() {
-        return this.messageDigest.digest();
+        final byte[] res = new byte[digest.getDigestSize()];
+
+        digest.doFinal(res, 0);
+
+        return res;
     }
 
     @Override
     public String toString() {
         return new ToStringBuilder(this)
-                .append("messageDigest", MantaUtils.byteArrayAsHexString(getDigest()))
+                .append("digest", MantaUtils.byteArrayAsHexString(getDigest()))
                 .append("wrapped", wrapped)
                 .toString();
     }