From af9bfd3e2107753de11ec61f80fc8c5aade5a20c Mon Sep 17 00:00:00 2001 From: Ryan Moore Date: Thu, 9 Oct 2014 11:44:00 -0600 Subject: [PATCH 1/4] Added support for a customer to ignore SSL Certificate Verification for testing. --- .../ds3client/integration/Util.java | 20 +++++++-- .../integration/BucketIntegration_Test.java | 9 ++-- .../ds3client/integration/Insecure_Test.java | 36 ++++++++++++++++ .../ds3client/ConnectionDetailsImpl.java | 29 ++++++++++--- .../ds3client/Ds3ClientBuilder.java | 15 +++++-- .../ds3client/NetworkClientImpl.java | 43 ++++++++++++++++--- .../ds3client/helpers/JobState.java | 2 +- .../ds3client/helpers/ReadJobImpl.java | 4 +- .../ds3client/helpers/WriteJobImpl.java | 10 +++-- .../networking/ConnectionDetails.java | 12 +++++- .../ds3client/networking/NetUtils.java | 2 +- .../ds3client/utils/SSLSetupException.java | 7 +++ .../ds3client/samples/BulkPutExample.java | 2 +- .../ds3client/samples/Ds3BulkGetExample.java | 2 +- .../samples/Ds3ServiceListExample.java | 2 +- .../ds3client/Ds3ClientBuilder_Test.java | 10 ++--- .../ds3client/Ds3Client_Test.java | 2 +- .../helpers/Ds3ClientHelpers_Test.java | 23 +++++----- 18 files changed, 176 insertions(+), 54 deletions(-) create mode 100644 integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java diff --git a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index b800defa4..f184b9d46 100644 --- a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -10,6 +10,7 @@ import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.serializer.XmlProcessingException; import com.spectralogic.ds3client.utils.ResourceUtils; +import com.spectralogic.ds3client.utils.SSLSetupException; import java.io.File; import java.io.IOException; @@ -22,6 +23,18 @@ public class Util { private Util() {} public static Ds3Client fromEnv() { + final Ds3ClientBuilder builder = clientBuilder(); + builder.withHttps(false); + return builder.build(); + } + + public static Ds3Client insecureFromEnv() { + final Ds3ClientBuilder builder = clientBuilder(); + builder.withSecure(false); + return builder.build(); + } + + private static Ds3ClientBuilder clientBuilder() { final String endpoint = System.getenv("DS3_ENDPOINT"); final String accessKey = System.getenv("DS3_ACCESS_KEY"); final String secretKey = System.getenv("DS3_SECRET_KEY"); @@ -40,15 +53,14 @@ public static Ds3Client fromEnv() { } final Ds3ClientBuilder builder = Ds3ClientBuilder.create(endpoint,new Credentials(accessKey, secretKey)); - builder.withHttpSecure(false); if (httpProxy != null) { builder.withProxy(httpProxy); } - return builder.build(); + return builder; } private static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"}; - public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException { + public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException, SSLSetupException { final String resourceBaseName = "books/"; final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); @@ -66,7 +78,7 @@ public static void loadBookTestData(final Ds3Client client, final String bucketN .transfer(new ResourceObjectPutter(resourceBaseName)); } - public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException, SignatureException { + public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException, SignatureException, SSLSetupException { final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); final Iterable objects = helpers.listObjects(bucketName); diff --git a/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java b/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java index d17008c65..30b09ff29 100644 --- a/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java +++ b/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java @@ -5,6 +5,7 @@ import com.spectralogic.ds3client.models.ListBucketResult; import com.spectralogic.ds3client.serializer.XmlProcessingException; +import com.spectralogic.ds3client.utils.SSLSetupException; import org.junit.BeforeClass; import org.junit.Test; @@ -26,7 +27,7 @@ public static void startup() { } @Test - public void createBucket() throws IOException, SignatureException { + public void createBucket() throws IOException, SignatureException, SSLSetupException { final String bucketName = "test_create_bucket"; client.putBucket(new PutBucketRequest(bucketName)); @@ -43,7 +44,7 @@ public void createBucket() throws IOException, SignatureException { } @Test - public void deleteBucket() throws IOException, SignatureException { + public void deleteBucket() throws IOException, SignatureException, SSLSetupException { final String bucketName = "test_delete_bucket"; client.putBucket(new PutBucketRequest(bucketName)); @@ -57,7 +58,7 @@ public void deleteBucket() throws IOException, SignatureException { } @Test - public void emptyBucket() throws IOException, SignatureException { + public void emptyBucket() throws IOException, SignatureException, SSLSetupException { final String bucketName = "test_empty_bucket"; try { @@ -74,7 +75,7 @@ public void emptyBucket() throws IOException, SignatureException { } @Test - public void listContents() throws IOException, SignatureException, XmlProcessingException, URISyntaxException { + public void listContents() throws IOException, SignatureException, XmlProcessingException, URISyntaxException, SSLSetupException { final String bucketName = "test_contents_bucket"; try { diff --git a/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java new file mode 100644 index 000000000..aab56b1d1 --- /dev/null +++ b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java @@ -0,0 +1,36 @@ +package com.spectralogic.ds3client.integration; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.commands.GetServiceRequest; +import com.spectralogic.ds3client.commands.GetServiceResponse; +import com.spectralogic.ds3client.utils.SSLSetupException; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.security.SignatureException; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; + +/** + * + * This test is intended to be a sanity check to make sure that we can successfully ignore ssl certificate validation. + * + */ +public class Insecure_Test { + + private static Ds3Client client; + + @BeforeClass + public static void startup() { + client = Util.insecureFromEnv(); + } + + @Test + public void getService() throws SignatureException, IOException, SSLSetupException { + final GetServiceResponse response = client.getService(new GetServiceRequest()); + + assertThat(response, is(notNullValue())); + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java index bcbeb52f2..d83126316 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java @@ -25,18 +25,19 @@ static class Builder implements com.spectralogic.ds3client.utils.Builder { final private String endpoint; final private Credentials credentials; + private boolean https = true; private boolean secure = true; private URI proxy = null; private int retries = 5; @@ -49,8 +50,8 @@ public static Ds3ClientBuilder create(final String endpoint, final Credentials c * @param secure True will use HTTPS, false will use HTTP. * @return The current builder. */ - public Ds3ClientBuilder withHttpSecure(final boolean secure) { - this.secure = secure; + public Ds3ClientBuilder withHttps(final boolean secure) { + this.https = secure; return this; } @@ -63,6 +64,14 @@ public Ds3ClientBuilder withBufferSize(final int bufferSize) { return this; } + /** + * Specifies if the library should perform SSL certificate validation. + */ + public Ds3ClientBuilder withSecure(final boolean secure) { + this.secure = secure; + return this; + } + /** * Sets a HTTP proxy. * @param proxy The endpoint of the HTTP proxy. @@ -102,7 +111,7 @@ public Ds3ClientBuilder withRedirectRetries(final int retries) { @Override public Ds3Client build() { final ConnectionDetailsImpl.Builder connBuilder = ConnectionDetailsImpl.builder(this.endpoint, this.credentials) - .withProxy(this.proxy).withSecure(this.secure).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); + .withProxy(this.proxy).withHttps(this.https).withSecure(this.secure).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); final NetworkClient netClient = new NetworkClientImpl(connBuilder.build()); return new Ds3ClientImpl(netClient); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java index 574e0a3d9..903388d14 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java @@ -20,6 +20,7 @@ import com.spectralogic.ds3client.models.SignatureDetails; import com.spectralogic.ds3client.networking.*; import com.spectralogic.ds3client.utils.DateFormatter; +import com.spectralogic.ds3client.utils.SSLSetupException; import com.spectralogic.ds3client.utils.Signature; import org.apache.http.HttpHost; @@ -28,17 +29,22 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ssl.*; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicHttpEntityEnclosingRequest; import org.apache.http.message.BasicHttpRequest; +import javax.net.ssl.SSLContext; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; -import java.security.SignatureException; +import java.security.*; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.Map; class NetworkClientImpl implements NetworkClient { @@ -100,7 +106,7 @@ public CloseableHttpResponse execute() throws IOException, SignatureException { final HttpRequest httpRequest = this.buildHttpRequest(); this.addHeaders(httpRequest); - return HttpClients.createDefault().execute(this.host, httpRequest, this.getContext()); + return getClient().execute(this.host, httpRequest, this.getContext()); } private HttpHost buildHost() throws MalformedURLException { @@ -113,12 +119,39 @@ private HttpHost buildHost() throws MalformedURLException { } } + private CloseableHttpClient getClient() { + if (!NetworkClientImpl.this.getConnectionDetails().isHttps()) { + return HttpClients.createDefault(); + } + + try { + + final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { + @Override + public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + return true; + } + }).useTLS().build(); + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new AllowAllHostnameVerifier()); + return HttpClients.custom().setSSLSocketFactory( + sslsf).build(); + + } catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new SSLSetupException(e); + } + } + private int getPort(final URL url) { final int port = url.getPort(); - if(port < 0) { - return 80; + if(port > 0) { + return port; + } + + if (url.getProtocol().equals("https")) { + return 443; } - return port; + return 80; } private HttpRequest buildHttpRequest() throws IOException { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 5179e6bf1..13cf10b26 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -56,7 +56,7 @@ private static int getObjectCount(final Collection chunks) { private static AutoCloseableCache buildCache( final ObjectChannelBuilder channelBuilder) { - return new AutoCloseableCache( + return new AutoCloseableCache<>( new ValueBuilder() { @Override public WindowedChannelFactory get(final String key) { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index c15f3bce7..a7d187a18 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -46,9 +46,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder) while (jobState.hasObjects()) { transferNextChunks(chunkTransferrer); } - } catch (final SignatureException | IOException | XmlProcessingException e) { - throw e; - } catch (final RuntimeException e) { + } catch (final RuntimeException | SignatureException | IOException | XmlProcessingException e) { throw e; } catch (final Exception e) { throw new RuntimeException(e); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 1ebf8c67c..70fe0b5e0 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -53,9 +53,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder) for (final Objects chunk : filteredChunks) { chunkTransferrer.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk)))); } - } catch (final SignatureException | IOException | XmlProcessingException e) { - throw e; - } catch (final RuntimeException e) { + } catch (final SignatureException | IOException | XmlProcessingException | RuntimeException e) { throw e; } catch (final Exception e) { throw new RuntimeException(e); @@ -88,6 +86,12 @@ private Objects tryAllocateChunk(final Objects filtered) throws IOException, Sig } } + /** + * Filters out chunks that have already been completed. We will get the same chunk name back from the server, but it + * will not have any objects in it, so we remove that from the list of objects that are returned. + * @param chunks The list to be filtered + * @return The filtered list + */ private static List filterChunks(final List chunks) { final List filteredChunks = new ArrayList<>(); for (final Objects chunk : chunks) { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java index 0c3ddd6ae..7e37f4e77 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java @@ -24,11 +24,21 @@ public interface ConnectionDetails { public Credentials getCredentials(); - public boolean isSecure(); + /** + * If true the network layer will use Https. + * @return + */ + public boolean isHttps(); public URI getProxy(); public int getRetries(); public int getBufferSize(); + + /** + * Returns true if the network layer should perform certificate authentication for SSL. False will disable + * certificate authentication. + */ + boolean isSecure(); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java index 1b3012ccf..b8078c1c1 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java @@ -38,7 +38,7 @@ public static URL buildUrl(final ConnectionDetails connectionDetails, final Stri public static URL buildUrl(final ConnectionDetails connectionDetails, final String path, final Map params) throws MalformedURLException { final StringBuilder builder = new StringBuilder(); - builder.append(connectionDetails.isSecure()? "https": "http").append("://"); + builder.append(connectionDetails.isHttps()? "https": "http").append("://"); builder.append(connectionDetails.getEndpoint()); if(!path.startsWith("/")) { builder.append('/'); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java new file mode 100644 index 000000000..f67525e46 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java @@ -0,0 +1,7 @@ +package com.spectralogic.ds3client.utils; + +public class SSLSetupException extends RuntimeException { + public SSLSetupException(final Exception e) { + super(e); + } +} diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java index 719550bc4..5404eaf8d 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java @@ -18,7 +18,7 @@ public class BulkPutExample { public static void main(final String args[]) throws IOException, SignatureException, XmlProcessingException { final Ds3Client client = Ds3ClientBuilder.create("endpoint:8080", new Credentials("accessId", "secretKey")) - .withHttpSecure(false) + .withHttps(false) .build(); // Wrap the Ds3Client with the helper functions diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java index dc6043860..6220e9b03 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java @@ -27,7 +27,7 @@ public static void main(final String args[]) throws IOException, SignatureExcept // Get a client builder and then build a client instance. This is the main entry point to the SDK. final Ds3Client client = Ds3ClientBuilder.create("ds3Endpoint:8080", - new Credentials("accessKey", "secretKey")).withHttpSecure(false).build(); + new Credentials("accessKey", "secretKey")).withHttps(false).build(); final String bucket = "bucketName"; //The bucket we are interested in getting objects from. diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java index fd4e901c7..03a179d5a 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java @@ -16,7 +16,7 @@ public static void main(final String args[]) throws IOException, SignatureExcept // Get a client builder and then build a client instance. This is the main entry point to the SDK. final Ds3Client client = Ds3ClientBuilder.create("ds3Endpoint:8080", - new Credentials("accessKey", "secretKey")).withHttpSecure(false).build(); + new Credentials("accessKey", "secretKey")).withHttps(false).build(); // Tell the client to get us a list of all buckets, this is called a service list. final GetServiceResponse response = client.getService(new GetServiceRequest()); diff --git a/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java index bb9e29d0c..a8e7f3b81 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java @@ -33,22 +33,22 @@ public void createBasicClient() throws Exception { @Test public void isNotSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); - final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttpSecure(false).build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(false)); + final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttps(false).build(); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(false)); } @Test public void defaultSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); final Ds3ClientImpl client = (Ds3ClientImpl)builder.build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(true)); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(true)); } @Test public void isSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); - final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttpSecure(true).build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(true)); + final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttps(true).build(); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(true)); } @Test diff --git a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java index 87395cea6..8debbe04c 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java @@ -528,7 +528,7 @@ private static void checkJob( } @Test - public void getJob() throws SignatureException, IOException { + public void getJob() throws SignatureException, IOException{ checkMasterObjectList( MockNetwork .expecting(HttpVerb.GET, "/_rest_/job/1a85e743-ec8f-4789-afec-97e587a26936", null, null) diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 831dee6ac..8dc98b7c0 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -89,7 +89,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { } } - @Test + @Test(expected = StubException.class) public void testReadObjectsWithFailedGet() throws SignatureException, IOException, XmlProcessingException { final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); @@ -111,17 +111,14 @@ public void testReadObjectsWithFailedGet() throws SignatureException, IOExceptio new Ds3Object("baz") )); - try { - job.transfer(new ObjectChannelBuilder() { - @Override - public SeekableByteChannel buildChannel(final String key) throws IOException { - // We don't care about the contents since we just want to know that the exception handling works correctly. - return new ByteArraySeekableByteChannel(); - } - }); - Assert.fail("Should have failed with an exception before we got here."); - } catch (final StubException e) { - } + job.transfer(new ObjectChannelBuilder() { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + // We don't care about the contents since we just want to know that the exception handling works correctly. + return new ByteArraySeekableByteChannel(); + } + }); + Assert.fail("Should have failed with an exception before we got here."); } @Test @@ -187,7 +184,7 @@ public void testWriteObjectsWithFailedPut() throws SignatureException, IOExcepti final PutObjectResponse putResponse = Mockito.mock(PutObjectResponse.class); Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 0, "foo co"))).thenThrow(new StubException()); Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "baz", jobId, 0, "baz co"))).thenReturn(putResponse); - + final Job job = Ds3ClientHelpers.wrap(ds3Client).startWriteJob(MYBUCKET, Lists.newArrayList( new Ds3Object("foo"), new Ds3Object("bar"), From 45ac763cc6d89584f67068de23b4c0dc7643faf1 Mon Sep 17 00:00:00 2001 From: Ryan Moore Date: Thu, 9 Oct 2014 11:46:40 -0600 Subject: [PATCH 2/4] Removing some unneeded checked exceptions. --- .../com/spectralogic/ds3client/integration/Util.java | 5 ++--- .../ds3client/integration/BucketIntegration_Test.java | 9 ++++----- .../ds3client/integration/Insecure_Test.java | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index f184b9d46..0ec105b13 100644 --- a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -10,7 +10,6 @@ import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.serializer.XmlProcessingException; import com.spectralogic.ds3client.utils.ResourceUtils; -import com.spectralogic.ds3client.utils.SSLSetupException; import java.io.File; import java.io.IOException; @@ -60,7 +59,7 @@ private static Ds3ClientBuilder clientBuilder() { } private static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"}; - public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException, SSLSetupException { + public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException { final String resourceBaseName = "books/"; final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); @@ -78,7 +77,7 @@ public static void loadBookTestData(final Ds3Client client, final String bucketN .transfer(new ResourceObjectPutter(resourceBaseName)); } - public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException, SignatureException, SSLSetupException { + public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException, SignatureException { final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client); final Iterable objects = helpers.listObjects(bucketName); diff --git a/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java b/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java index 30b09ff29..d17008c65 100644 --- a/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java +++ b/integration/src/test/java/com/spectralogic/ds3client/integration/BucketIntegration_Test.java @@ -5,7 +5,6 @@ import com.spectralogic.ds3client.models.ListBucketResult; import com.spectralogic.ds3client.serializer.XmlProcessingException; -import com.spectralogic.ds3client.utils.SSLSetupException; import org.junit.BeforeClass; import org.junit.Test; @@ -27,7 +26,7 @@ public static void startup() { } @Test - public void createBucket() throws IOException, SignatureException, SSLSetupException { + public void createBucket() throws IOException, SignatureException { final String bucketName = "test_create_bucket"; client.putBucket(new PutBucketRequest(bucketName)); @@ -44,7 +43,7 @@ public void createBucket() throws IOException, SignatureException, SSLSetupExcep } @Test - public void deleteBucket() throws IOException, SignatureException, SSLSetupException { + public void deleteBucket() throws IOException, SignatureException { final String bucketName = "test_delete_bucket"; client.putBucket(new PutBucketRequest(bucketName)); @@ -58,7 +57,7 @@ public void deleteBucket() throws IOException, SignatureException, SSLSetupExcep } @Test - public void emptyBucket() throws IOException, SignatureException, SSLSetupException { + public void emptyBucket() throws IOException, SignatureException { final String bucketName = "test_empty_bucket"; try { @@ -75,7 +74,7 @@ public void emptyBucket() throws IOException, SignatureException, SSLSetupExcept } @Test - public void listContents() throws IOException, SignatureException, XmlProcessingException, URISyntaxException, SSLSetupException { + public void listContents() throws IOException, SignatureException, XmlProcessingException, URISyntaxException { final String bucketName = "test_contents_bucket"; try { diff --git a/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java index aab56b1d1..7787880cb 100644 --- a/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java +++ b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java @@ -3,7 +3,6 @@ import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.commands.GetServiceRequest; import com.spectralogic.ds3client.commands.GetServiceResponse; -import com.spectralogic.ds3client.utils.SSLSetupException; import org.junit.BeforeClass; import org.junit.Test; @@ -28,7 +27,7 @@ public static void startup() { } @Test - public void getService() throws SignatureException, IOException, SSLSetupException { + public void getService() throws SignatureException, IOException{ final GetServiceResponse response = client.getService(new GetServiceRequest()); assertThat(response, is(notNullValue())); From 87d09314cd0a875bc537f9dbf9b2b303367110dd Mon Sep 17 00:00:00 2001 From: Ryan Moore Date: Thu, 9 Oct 2014 12:07:56 -0600 Subject: [PATCH 3/4] Refactored out some port returning code into a static helper function. Reved the version number. --- build.gradle | 2 +- .../ds3client/NetworkClientImpl.java | 14 +--------- .../ds3client/networking/NetUtils.java | 13 ++++++++++ .../ds3client/utils/NetUtils_Test.java | 26 +++++++++++++++++-- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/build.gradle b/build.gradle index 4f9811cf1..7e12f9427 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ allprojects { group = 'com.spectralogic' - version = '0.7.4-SNAPSHOT' + version = '0.7.5-SNAPSHOT' } subprojects { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java index 903388d14..bac62fa41 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java @@ -115,7 +115,7 @@ private HttpHost buildHost() throws MalformedURLException { return new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme()); } else { final URL url = NetUtils.buildUrl(NetworkClientImpl.this.connectionDetails, "/"); - return new HttpHost(url.getHost(), this.getPort(url), url.getProtocol()); + return new HttpHost(url.getHost(), NetUtils.getPort(url), url.getProtocol()); } } @@ -142,18 +142,6 @@ public boolean isTrusted(final X509Certificate[] chain, final String authType) t } } - private int getPort(final URL url) { - final int port = url.getPort(); - if(port > 0) { - return port; - } - - if (url.getProtocol().equals("https")) { - return 443; - } - return 80; - } - private HttpRequest buildHttpRequest() throws IOException { final String verb = this.ds3Request.getVerb().toString(); final String path = this.buildPath(); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java index b8078c1c1..d7514a156 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java @@ -115,4 +115,17 @@ private static String bucketPath(final String bucket) { public static String buildHostField(final ConnectionDetails details) { return details.getEndpoint(); } + + public static int getPort(final URL url) { + final int port = url.getPort(); + if(port > 0) { + return port; + } + + if (url.getProtocol().equals("https")) { + return 443; + } + return 80; + } + } diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java index fc4e83045..eef4c22ae 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java @@ -15,9 +15,9 @@ package com.spectralogic.ds3client.utils; - import com.spectralogic.ds3client.BulkCommand; import com.spectralogic.ds3client.ConnectionFixture; +import com.spectralogic.ds3client.networking.ConnectionDetails; import com.spectralogic.ds3client.networking.NetUtils; import org.junit.Test; @@ -26,8 +26,9 @@ import java.util.HashMap; import java.util.Map; -import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; public class NetUtils_Test { @@ -143,4 +144,25 @@ public void escapeSpacesInQueryParam() throws MalformedURLException { final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "path", queryParams); assertThat(result.toString(), is("http://localhost:8080/path?foo=bar%20val")); } + + @Test + public void getPortBack() throws MalformedURLException { + final URL url = new URL("http://localhost:8080/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 8080); + } + + @Test + public void getHttpsDefaultPort() throws MalformedURLException { + final URL url = new URL("https://localhost/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 443); + } + + @Test + public void getHttpDefaultPort() throws MalformedURLException { + final URL url = new URL("http://localhost/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 80); + } } From bc2f40ac42f63c146282f44e6d22d7bfa35794a1 Mon Sep 17 00:00:00 2001 From: Ryan Moore Date: Mon, 13 Oct 2014 11:47:29 -0600 Subject: [PATCH 4/4] Changed withSecure to withCertificateVerification. Also adjusted the structure of the certificate verification code so that it is easier to reason about when first looking at it. --- .../ds3client/integration/Util.java | 2 +- .../ds3client/ConnectionDetailsImpl.java | 14 +++---- .../ds3client/Ds3ClientBuilder.java | 8 ++-- .../ds3client/NetworkClientImpl.java | 37 ++++++++++--------- .../networking/ConnectionDetails.java | 2 +- .../helpers/Ds3ClientHelpers_Test.java | 1 - 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index 0ec105b13..03ced8c03 100644 --- a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -29,7 +29,7 @@ public static Ds3Client fromEnv() { public static Ds3Client insecureFromEnv() { final Ds3ClientBuilder builder = clientBuilder(); - builder.withSecure(false); + builder.withCertificateVerification(false); return builder.build(); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java index d83126316..3158af903 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java @@ -29,7 +29,7 @@ static class Builder implements com.spectralogic.ds3client.utils.Builder { final private Credentials credentials; private boolean https = true; - private boolean secure = true; + private boolean certificateVerification = true; private URI proxy = null; private int retries = 5; private int bufferSize = 1024 * 1024; @@ -67,8 +67,8 @@ public Ds3ClientBuilder withBufferSize(final int bufferSize) { /** * Specifies if the library should perform SSL certificate validation. */ - public Ds3ClientBuilder withSecure(final boolean secure) { - this.secure = secure; + public Ds3ClientBuilder withCertificateVerification(final boolean certificateVerification) { + this.certificateVerification = certificateVerification; return this; } @@ -111,7 +111,7 @@ public Ds3ClientBuilder withRedirectRetries(final int retries) { @Override public Ds3Client build() { final ConnectionDetailsImpl.Builder connBuilder = ConnectionDetailsImpl.builder(this.endpoint, this.credentials) - .withProxy(this.proxy).withHttps(this.https).withSecure(this.secure).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); + .withProxy(this.proxy).withHttps(this.https).withCertificateVerification(this.certificateVerification).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); final NetworkClient netClient = new NetworkClientImpl(connBuilder.build()); return new Ds3ClientImpl(netClient); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java index bac62fa41..fd68fad4b 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java @@ -120,25 +120,26 @@ private HttpHost buildHost() throws MalformedURLException { } private CloseableHttpClient getClient() { - if (!NetworkClientImpl.this.getConnectionDetails().isHttps()) { - return HttpClients.createDefault(); + if (NetworkClientImpl.this.getConnectionDetails().isHttps() && !NetworkClientImpl.this.getConnectionDetails().isCertificateVerification()) { + try { + + final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { + @Override + public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + return true; + } + }).useTLS().build(); + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new AllowAllHostnameVerifier()); + return HttpClients.custom().setSSLSocketFactory( + sslsf).build(); + + } catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new SSLSetupException(e); + } } - - try { - - final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { - @Override - public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { - return true; - } - }).useTLS().build(); - - final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new AllowAllHostnameVerifier()); - return HttpClients.custom().setSSLSocketFactory( - sslsf).build(); - - } catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { - throw new SSLSetupException(e); + else { + return HttpClients.createDefault(); } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java index 7e37f4e77..b2c83522f 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java @@ -40,5 +40,5 @@ public interface ConnectionDetails { * Returns true if the network layer should perform certificate authentication for SSL. False will disable * certificate authentication. */ - boolean isSecure(); + boolean isCertificateVerification(); } diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 8dc98b7c0..72a3122c5 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -118,7 +118,6 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { return new ByteArraySeekableByteChannel(); } }); - Assert.fail("Should have failed with an exception before we got here."); } @Test