Adds better handling for HTTPS

build.gradle

@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic'
-    version = '0.7.4-SNAPSHOT'
+    version = '0.7.5-SNAPSHOT'
 }
 
 subprojects {

integration/src/main/java/com/spectralogic/ds3client/integration/Util.java

@@ -22,6 +22,18 @@ public class Util {
     private Util() {}
 
     public static Ds3Client fromEnv() {
+        final Ds3ClientBuilder builder = clientBuilder();
+        builder.withHttps(false);
+        return builder.build();
+    }
+
+    public static Ds3Client insecureFromEnv() {
+        final Ds3ClientBuilder builder = clientBuilder();
+        builder.withCertificateVerification(false);
+        return builder.build();
+    }
+
+    private static Ds3ClientBuilder clientBuilder() {
         final String endpoint = System.getenv("DS3_ENDPOINT");
         final String accessKey = System.getenv("DS3_ACCESS_KEY");
         final String secretKey = System.getenv("DS3_SECRET_KEY");
@@ -40,11 +52,10 @@ public static Ds3Client fromEnv() {
         }
 
         final Ds3ClientBuilder builder = Ds3ClientBuilder.create(endpoint,new Credentials(accessKey, secretKey));
-        builder.withHttpSecure(false);
         if (httpProxy != null) {
             builder.withProxy(httpProxy);
         }
-        return builder.build();
+        return builder;
     }
 
     private static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"};

integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java

@@ -0,0 +1,35 @@
+package com.spectralogic.ds3client.integration;
+
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.GetServiceRequest;
+import com.spectralogic.ds3client.commands.GetServiceResponse;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.security.SignatureException;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.assertThat;
+
+/**
+ *
+ * This test is intended to be a sanity check to make sure that we can successfully ignore ssl certificate validation.
+ *
+ */
+public class Insecure_Test {
+
+    private static Ds3Client client;
+
+    @BeforeClass
+    public static void startup() {
+        client = Util.insecureFromEnv();
+    }
+
+    @Test
+    public void getService() throws SignatureException, IOException{
+        final GetServiceResponse response = client.getService(new GetServiceRequest());
+
+        assertThat(response, is(notNullValue()));
+    }
+}

sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java

@@ -25,18 +25,19 @@ static class Builder implements com.spectralogic.ds3client.utils.Builder<Connect
 
         private final String endpoint;
         private final Credentials credentials;
-        private boolean secure = false;
+        private boolean https = false;
         private URI proxy = null;
         private int retries = 5;
         private int bufferSize = 1024 * 1024;
+        private boolean certificateVerification;
 
         private Builder(final String endpoint, final Credentials credentials) {
             this.endpoint = endpoint;
             this.credentials = credentials;
         }
 
-        public Builder withSecure(final boolean secure) {
-            this.secure = secure;
+        public Builder withHttps(final boolean secure) {
+            this.https = secure;
             return this;
         }
 
@@ -55,18 +56,25 @@ public Builder withBufferSize(final int bufferSize) {
             return this;
         }
 
+        public Builder withCertificateVerification(final boolean certificateVerification) {
+            this.certificateVerification = certificateVerification;
+            return this;
+        }
+
         @Override
         public ConnectionDetailsImpl build() {
             return new ConnectionDetailsImpl(this);
         }
+
     }
 
     private final String endpoint;
     private final Credentials credentials;
-    private final boolean secure;
+    private final boolean https;
     private final URI proxy;
     private final int retries;
     private final int bufferSize;
+    private final boolean certificateVerification;
 
     static Builder builder(final String uriEndpoint, final Credentials credentials) {
         return new Builder(uriEndpoint, credentials);
@@ -75,10 +83,11 @@ static Builder builder(final String uriEndpoint, final Credentials credentials)
     private ConnectionDetailsImpl(final Builder builder) {
         this.endpoint = builder.endpoint;
         this.credentials = builder.credentials;
-        this.secure = builder.secure;
+        this.https = builder.https;
         this.proxy = builder.proxy;
         this.retries = builder.retries;
         this.bufferSize = builder.bufferSize;
+        this.certificateVerification = builder.certificateVerification;
     }
 
     @Override
@@ -92,8 +101,8 @@ public Credentials getCredentials() {
     }
 
     @Override
-    public boolean isSecure() {
-        return secure;
+    public boolean isHttps() {
+        return https;
     }
 
     @Override
@@ -110,4 +119,10 @@ public int getRetries() {
     public int getBufferSize() {
         return bufferSize;
     }
+
+    @Override
+    public boolean isCertificateVerification() {
+        return certificateVerification;
+    }
+
 }

sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientBuilder.java

@@ -18,7 +18,8 @@ public class Ds3ClientBuilder implements Builder<Ds3Client> {
     final private String endpoint;
     final private Credentials credentials;
 
-    private boolean secure = true;
+    private boolean https = true;
+    private boolean certificateVerification = true;
     private URI proxy = null;
     private int retries = 5;
     private int bufferSize = 1024 * 1024;
@@ -49,8 +50,8 @@ public static Ds3ClientBuilder create(final String endpoint, final Credentials c
      * @param secure True will use HTTPS, false will use HTTP.
      * @return The current builder.
      */
-    public Ds3ClientBuilder withHttpSecure(final boolean secure) {
-        this.secure = secure;
+    public Ds3ClientBuilder withHttps(final boolean secure) {
+        this.https = secure;
         return this;
     }
 
@@ -63,6 +64,14 @@ public Ds3ClientBuilder withBufferSize(final int bufferSize) {
         return this;
     }
 
+    /**
+     * Specifies if the library should perform SSL certificate validation.
+     */
+    public Ds3ClientBuilder withCertificateVerification(final boolean certificateVerification) {
+        this.certificateVerification = certificateVerification;
+        return this;
+    }
+
     /**
      * Sets a HTTP proxy.
      * @param proxy The endpoint of the HTTP proxy.
@@ -102,7 +111,7 @@ public Ds3ClientBuilder withRedirectRetries(final int retries) {
     @Override
     public Ds3Client build() {
         final ConnectionDetailsImpl.Builder connBuilder = ConnectionDetailsImpl.builder(this.endpoint, this.credentials)
-            .withProxy(this.proxy).withSecure(this.secure).withRedirectRetries(this.retries).withBufferSize(this.bufferSize);
+            .withProxy(this.proxy).withHttps(this.https).withCertificateVerification(this.certificateVerification).withRedirectRetries(this.retries).withBufferSize(this.bufferSize);
 
         final NetworkClient netClient = new NetworkClientImpl(connBuilder.build());
         return new Ds3ClientImpl(netClient);

sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java

@@ -20,6 +20,7 @@
 import com.spectralogic.ds3client.models.SignatureDetails;
 import com.spectralogic.ds3client.networking.*;
 import com.spectralogic.ds3client.utils.DateFormatter;
+import com.spectralogic.ds3client.utils.SSLSetupException;
 import com.spectralogic.ds3client.utils.Signature;
 
 import org.apache.http.HttpHost;
@@ -28,17 +29,22 @@
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.ssl.*;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.message.BasicHttpEntityEnclosingRequest;
 import org.apache.http.message.BasicHttpRequest;
 
+import javax.net.ssl.SSLContext;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
-import java.security.SignatureException;
+import java.security.*;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
 import java.util.Map;
 
 class NetworkClientImpl implements NetworkClient {
@@ -100,7 +106,7 @@ public CloseableHttpResponse execute() throws IOException, SignatureException {
 
             final HttpRequest httpRequest = this.buildHttpRequest();
             this.addHeaders(httpRequest);
-            return HttpClients.createDefault().execute(this.host, httpRequest, this.getContext());
+            return getClient().execute(this.host, httpRequest, this.getContext());
         }
 
         private HttpHost buildHost() throws MalformedURLException {
@@ -109,16 +115,32 @@ private HttpHost buildHost() throws MalformedURLException {
                 return new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme());
             } else {
                 final URL url = NetUtils.buildUrl(NetworkClientImpl.this.connectionDetails, "/");
-                return new HttpHost(url.getHost(), this.getPort(url), url.getProtocol());
+                return new HttpHost(url.getHost(), NetUtils.getPort(url), url.getProtocol());
             }
         }
 
-        private int getPort(final URL url) {
-            final int port = url.getPort();
-            if(port < 0) {
-                return 80;
+        private CloseableHttpClient getClient() {
+            if (NetworkClientImpl.this.getConnectionDetails().isHttps() && !NetworkClientImpl.this.getConnectionDetails().isCertificateVerification()) {
+                try {
+
+                    final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
+                        @Override
+                        public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException {
+                            return true;
+                        }
+                    }).useTLS().build();
+
+                    final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new AllowAllHostnameVerifier());
+                    return HttpClients.custom().setSSLSocketFactory(
+                            sslsf).build();
+
+                } catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {
+                    throw new SSLSetupException(e);
+                }
+            }
+            else {
+                return HttpClients.createDefault();
             }
-            return port;
         }
 
         private HttpRequest buildHttpRequest() throws IOException {

sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java

@@ -56,7 +56,7 @@ private static int getObjectCount(final Collection<Objects> chunks) {
 
     private static AutoCloseableCache<String, WindowedChannelFactory> buildCache(
             final ObjectChannelBuilder channelBuilder) {
-        return new AutoCloseableCache<String, WindowedChannelFactory>(
+        return new AutoCloseableCache<>(
             new ValueBuilder<String, WindowedChannelFactory>() {
                 @Override
                 public WindowedChannelFactory get(final String key) {

sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java

@@ -46,9 +46,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder)
             while (jobState.hasObjects()) {
                 transferNextChunks(chunkTransferrer);
             }
-        } catch (final SignatureException | IOException | XmlProcessingException e) {
-            throw e;
-        } catch (final RuntimeException e) {
+        } catch (final RuntimeException | SignatureException | IOException | XmlProcessingException e) {
             throw e;
         } catch (final Exception e) {
             throw new RuntimeException(e);

sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java

@@ -53,9 +53,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder)
             for (final Objects chunk : filteredChunks) {
                 chunkTransferrer.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk))));
             }
-        } catch (final SignatureException | IOException | XmlProcessingException e) {
-            throw e;
-        } catch (final RuntimeException e) {
+        } catch (final SignatureException | IOException | XmlProcessingException | RuntimeException e) {
             throw e;
         } catch (final Exception e) {
             throw new RuntimeException(e);
@@ -88,6 +86,12 @@ private Objects tryAllocateChunk(final Objects filtered) throws IOException, Sig
         }
     }
 
+    /**
+     * Filters out chunks that have already been completed.  We will get the same chunk name back from the server, but it
+     * will not have any objects in it, so we remove that from the list of objects that are returned.
+     * @param chunks The list to be filtered
+     * @return The filtered list
+     */
     private static List<Objects> filterChunks(final List<Objects> chunks) {
         final List<Objects> filteredChunks = new ArrayList<>();
         for (final Objects chunk : chunks) {

sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java

@@ -24,11 +24,21 @@ public interface ConnectionDetails {
 
     public Credentials getCredentials();
 
-    public boolean isSecure();
+    /**
+     * If true the network layer will use Https.
+     * @return
+     */
+    public boolean isHttps();
 
     public URI getProxy();
 
     public int getRetries();
 
     public int getBufferSize();
+
+    /**
+     * Returns true if the network layer should perform certificate authentication for SSL.  False will disable
+     * certificate authentication.
+     */
+    boolean isCertificateVerification();
 }

sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java

@@ -38,7 +38,7 @@ public static URL buildUrl(final ConnectionDetails connectionDetails, final Stri
 
     public static URL buildUrl(final ConnectionDetails connectionDetails, final String path, final Map<String, String> params) throws MalformedURLException {
         final StringBuilder builder = new StringBuilder();
-        builder.append(connectionDetails.isSecure()? "https": "http").append("://");
+        builder.append(connectionDetails.isHttps()? "https": "http").append("://");
         builder.append(connectionDetails.getEndpoint());
         if(!path.startsWith("/")) {
             builder.append('/');
@@ -115,4 +115,17 @@ private static String bucketPath(final String bucket) {
     public static String buildHostField(final ConnectionDetails details) {
         return details.getEndpoint();
     }
+
+    public static int getPort(final URL url) {
+        final int port = url.getPort();
+        if(port > 0) {
+            return port;
+        }
+
+        if (url.getProtocol().equals("https")) {
+            return 443;
+        }
+        return 80;
+    }
+
 }

sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java

@@ -0,0 +1,7 @@
+package com.spectralogic.ds3client.utils;
+
+public class SSLSetupException extends RuntimeException {
+    public SSLSetupException(final Exception e) {
+        super(e);
+    }
+}

sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java

@@ -18,7 +18,7 @@ public class BulkPutExample {
     public static void main(final String args[]) throws IOException, SignatureException, XmlProcessingException {
         final Ds3Client client = Ds3ClientBuilder.create("endpoint:8080",
                 new Credentials("accessId", "secretKey"))
-                .withHttpSecure(false)
+                .withHttps(false)
                 .build();
 
         // Wrap the Ds3Client with the helper functions