diff --git a/build.gradle b/build.gradle index 4f9811cf1..7e12f9427 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ allprojects { group = 'com.spectralogic' - version = '0.7.4-SNAPSHOT' + version = '0.7.5-SNAPSHOT' } subprojects { diff --git a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java index b800defa4..03ced8c03 100644 --- a/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java +++ b/integration/src/main/java/com/spectralogic/ds3client/integration/Util.java @@ -22,6 +22,18 @@ public class Util { private Util() {} public static Ds3Client fromEnv() { + final Ds3ClientBuilder builder = clientBuilder(); + builder.withHttps(false); + return builder.build(); + } + + public static Ds3Client insecureFromEnv() { + final Ds3ClientBuilder builder = clientBuilder(); + builder.withCertificateVerification(false); + return builder.build(); + } + + private static Ds3ClientBuilder clientBuilder() { final String endpoint = System.getenv("DS3_ENDPOINT"); final String accessKey = System.getenv("DS3_ACCESS_KEY"); final String secretKey = System.getenv("DS3_SECRET_KEY"); @@ -40,11 +52,10 @@ public static Ds3Client fromEnv() { } final Ds3ClientBuilder builder = Ds3ClientBuilder.create(endpoint,new Credentials(accessKey, secretKey)); - builder.withHttpSecure(false); if (httpProxy != null) { builder.withProxy(httpProxy); } - return builder.build(); + return builder; } private static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"}; diff --git a/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java new file mode 100644 index 000000000..7787880cb --- /dev/null +++ b/integration/src/test/java/com/spectralogic/ds3client/integration/Insecure_Test.java @@ -0,0 +1,35 @@ +package com.spectralogic.ds3client.integration; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.commands.GetServiceRequest; +import com.spectralogic.ds3client.commands.GetServiceResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.security.SignatureException; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; + +/** + * + * This test is intended to be a sanity check to make sure that we can successfully ignore ssl certificate validation. + * + */ +public class Insecure_Test { + + private static Ds3Client client; + + @BeforeClass + public static void startup() { + client = Util.insecureFromEnv(); + } + + @Test + public void getService() throws SignatureException, IOException{ + final GetServiceResponse response = client.getService(new GetServiceRequest()); + + assertThat(response, is(notNullValue())); + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java index bcbeb52f2..3158af903 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/ConnectionDetailsImpl.java @@ -25,18 +25,19 @@ static class Builder implements com.spectralogic.ds3client.utils.Builder { final private String endpoint; final private Credentials credentials; - private boolean secure = true; + private boolean https = true; + private boolean certificateVerification = true; private URI proxy = null; private int retries = 5; private int bufferSize = 1024 * 1024; @@ -49,8 +50,8 @@ public static Ds3ClientBuilder create(final String endpoint, final Credentials c * @param secure True will use HTTPS, false will use HTTP. * @return The current builder. */ - public Ds3ClientBuilder withHttpSecure(final boolean secure) { - this.secure = secure; + public Ds3ClientBuilder withHttps(final boolean secure) { + this.https = secure; return this; } @@ -63,6 +64,14 @@ public Ds3ClientBuilder withBufferSize(final int bufferSize) { return this; } + /** + * Specifies if the library should perform SSL certificate validation. + */ + public Ds3ClientBuilder withCertificateVerification(final boolean certificateVerification) { + this.certificateVerification = certificateVerification; + return this; + } + /** * Sets a HTTP proxy. * @param proxy The endpoint of the HTTP proxy. @@ -102,7 +111,7 @@ public Ds3ClientBuilder withRedirectRetries(final int retries) { @Override public Ds3Client build() { final ConnectionDetailsImpl.Builder connBuilder = ConnectionDetailsImpl.builder(this.endpoint, this.credentials) - .withProxy(this.proxy).withSecure(this.secure).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); + .withProxy(this.proxy).withHttps(this.https).withCertificateVerification(this.certificateVerification).withRedirectRetries(this.retries).withBufferSize(this.bufferSize); final NetworkClient netClient = new NetworkClientImpl(connBuilder.build()); return new Ds3ClientImpl(netClient); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java index 574e0a3d9..fd68fad4b 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/NetworkClientImpl.java @@ -20,6 +20,7 @@ import com.spectralogic.ds3client.models.SignatureDetails; import com.spectralogic.ds3client.networking.*; import com.spectralogic.ds3client.utils.DateFormatter; +import com.spectralogic.ds3client.utils.SSLSetupException; import com.spectralogic.ds3client.utils.Signature; import org.apache.http.HttpHost; @@ -28,17 +29,22 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ssl.*; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicHttpEntityEnclosingRequest; import org.apache.http.message.BasicHttpRequest; +import javax.net.ssl.SSLContext; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; -import java.security.SignatureException; +import java.security.*; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.Map; class NetworkClientImpl implements NetworkClient { @@ -100,7 +106,7 @@ public CloseableHttpResponse execute() throws IOException, SignatureException { final HttpRequest httpRequest = this.buildHttpRequest(); this.addHeaders(httpRequest); - return HttpClients.createDefault().execute(this.host, httpRequest, this.getContext()); + return getClient().execute(this.host, httpRequest, this.getContext()); } private HttpHost buildHost() throws MalformedURLException { @@ -109,16 +115,32 @@ private HttpHost buildHost() throws MalformedURLException { return new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme()); } else { final URL url = NetUtils.buildUrl(NetworkClientImpl.this.connectionDetails, "/"); - return new HttpHost(url.getHost(), this.getPort(url), url.getProtocol()); + return new HttpHost(url.getHost(), NetUtils.getPort(url), url.getProtocol()); } } - private int getPort(final URL url) { - final int port = url.getPort(); - if(port < 0) { - return 80; + private CloseableHttpClient getClient() { + if (NetworkClientImpl.this.getConnectionDetails().isHttps() && !NetworkClientImpl.this.getConnectionDetails().isCertificateVerification()) { + try { + + final SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { + @Override + public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + return true; + } + }).useTLS().build(); + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new AllowAllHostnameVerifier()); + return HttpClients.custom().setSSLSocketFactory( + sslsf).build(); + + } catch (final NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new SSLSetupException(e); + } + } + else { + return HttpClients.createDefault(); } - return port; } private HttpRequest buildHttpRequest() throws IOException { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 5179e6bf1..13cf10b26 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -56,7 +56,7 @@ private static int getObjectCount(final Collection chunks) { private static AutoCloseableCache buildCache( final ObjectChannelBuilder channelBuilder) { - return new AutoCloseableCache( + return new AutoCloseableCache<>( new ValueBuilder() { @Override public WindowedChannelFactory get(final String key) { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index c15f3bce7..a7d187a18 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -46,9 +46,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder) while (jobState.hasObjects()) { transferNextChunks(chunkTransferrer); } - } catch (final SignatureException | IOException | XmlProcessingException e) { - throw e; - } catch (final RuntimeException e) { + } catch (final RuntimeException | SignatureException | IOException | XmlProcessingException e) { throw e; } catch (final Exception e) { throw new RuntimeException(e); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 1ebf8c67c..70fe0b5e0 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -53,9 +53,7 @@ public void transfer(final ObjectChannelBuilder channelBuilder) for (final Objects chunk : filteredChunks) { chunkTransferrer.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk)))); } - } catch (final SignatureException | IOException | XmlProcessingException e) { - throw e; - } catch (final RuntimeException e) { + } catch (final SignatureException | IOException | XmlProcessingException | RuntimeException e) { throw e; } catch (final Exception e) { throw new RuntimeException(e); @@ -88,6 +86,12 @@ private Objects tryAllocateChunk(final Objects filtered) throws IOException, Sig } } + /** + * Filters out chunks that have already been completed. We will get the same chunk name back from the server, but it + * will not have any objects in it, so we remove that from the list of objects that are returned. + * @param chunks The list to be filtered + * @return The filtered list + */ private static List filterChunks(final List chunks) { final List filteredChunks = new ArrayList<>(); for (final Objects chunk : chunks) { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java index 0c3ddd6ae..b2c83522f 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/ConnectionDetails.java @@ -24,11 +24,21 @@ public interface ConnectionDetails { public Credentials getCredentials(); - public boolean isSecure(); + /** + * If true the network layer will use Https. + * @return + */ + public boolean isHttps(); public URI getProxy(); public int getRetries(); public int getBufferSize(); + + /** + * Returns true if the network layer should perform certificate authentication for SSL. False will disable + * certificate authentication. + */ + boolean isCertificateVerification(); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java index 1b3012ccf..d7514a156 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/networking/NetUtils.java @@ -38,7 +38,7 @@ public static URL buildUrl(final ConnectionDetails connectionDetails, final Stri public static URL buildUrl(final ConnectionDetails connectionDetails, final String path, final Map params) throws MalformedURLException { final StringBuilder builder = new StringBuilder(); - builder.append(connectionDetails.isSecure()? "https": "http").append("://"); + builder.append(connectionDetails.isHttps()? "https": "http").append("://"); builder.append(connectionDetails.getEndpoint()); if(!path.startsWith("/")) { builder.append('/'); @@ -115,4 +115,17 @@ private static String bucketPath(final String bucket) { public static String buildHostField(final ConnectionDetails details) { return details.getEndpoint(); } + + public static int getPort(final URL url) { + final int port = url.getPort(); + if(port > 0) { + return port; + } + + if (url.getProtocol().equals("https")) { + return 443; + } + return 80; + } + } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java new file mode 100644 index 000000000..f67525e46 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/SSLSetupException.java @@ -0,0 +1,7 @@ +package com.spectralogic.ds3client.utils; + +public class SSLSetupException extends RuntimeException { + public SSLSetupException(final Exception e) { + super(e); + } +} diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java index 719550bc4..5404eaf8d 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/BulkPutExample.java @@ -18,7 +18,7 @@ public class BulkPutExample { public static void main(final String args[]) throws IOException, SignatureException, XmlProcessingException { final Ds3Client client = Ds3ClientBuilder.create("endpoint:8080", new Credentials("accessId", "secretKey")) - .withHttpSecure(false) + .withHttps(false) .build(); // Wrap the Ds3Client with the helper functions diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java index dc6043860..6220e9b03 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3BulkGetExample.java @@ -27,7 +27,7 @@ public static void main(final String args[]) throws IOException, SignatureExcept // Get a client builder and then build a client instance. This is the main entry point to the SDK. final Ds3Client client = Ds3ClientBuilder.create("ds3Endpoint:8080", - new Credentials("accessKey", "secretKey")).withHttpSecure(false).build(); + new Credentials("accessKey", "secretKey")).withHttps(false).build(); final String bucket = "bucketName"; //The bucket we are interested in getting objects from. diff --git a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java index fd4e901c7..03a179d5a 100644 --- a/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java +++ b/sdk/src/samples/java/com/spectralogic/ds3client/samples/Ds3ServiceListExample.java @@ -16,7 +16,7 @@ public static void main(final String args[]) throws IOException, SignatureExcept // Get a client builder and then build a client instance. This is the main entry point to the SDK. final Ds3Client client = Ds3ClientBuilder.create("ds3Endpoint:8080", - new Credentials("accessKey", "secretKey")).withHttpSecure(false).build(); + new Credentials("accessKey", "secretKey")).withHttps(false).build(); // Tell the client to get us a list of all buckets, this is called a service list. final GetServiceResponse response = client.getService(new GetServiceRequest()); diff --git a/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java index bb9e29d0c..a8e7f3b81 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/Ds3ClientBuilder_Test.java @@ -33,22 +33,22 @@ public void createBasicClient() throws Exception { @Test public void isNotSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); - final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttpSecure(false).build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(false)); + final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttps(false).build(); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(false)); } @Test public void defaultSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); final Ds3ClientImpl client = (Ds3ClientImpl)builder.build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(true)); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(true)); } @Test public void isSecure() throws Exception { final Ds3ClientBuilder builder = Ds3ClientBuilder.create("myEndPoint", new Credentials("foo","bar")); - final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttpSecure(true).build(); - assertThat(client.getNetClient().getConnectionDetails().isSecure(),is(true)); + final Ds3ClientImpl client = (Ds3ClientImpl)builder.withHttps(true).build(); + assertThat(client.getNetClient().getConnectionDetails().isHttps(),is(true)); } @Test diff --git a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java index 87395cea6..8debbe04c 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java @@ -528,7 +528,7 @@ private static void checkJob( } @Test - public void getJob() throws SignatureException, IOException { + public void getJob() throws SignatureException, IOException{ checkMasterObjectList( MockNetwork .expecting(HttpVerb.GET, "/_rest_/job/1a85e743-ec8f-4789-afec-97e587a26936", null, null) diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 831dee6ac..72a3122c5 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -89,7 +89,7 @@ public SeekableByteChannel buildChannel(final String key) throws IOException { } } - @Test + @Test(expected = StubException.class) public void testReadObjectsWithFailedGet() throws SignatureException, IOException, XmlProcessingException { final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); @@ -111,17 +111,13 @@ public void testReadObjectsWithFailedGet() throws SignatureException, IOExceptio new Ds3Object("baz") )); - try { - job.transfer(new ObjectChannelBuilder() { - @Override - public SeekableByteChannel buildChannel(final String key) throws IOException { - // We don't care about the contents since we just want to know that the exception handling works correctly. - return new ByteArraySeekableByteChannel(); - } - }); - Assert.fail("Should have failed with an exception before we got here."); - } catch (final StubException e) { - } + job.transfer(new ObjectChannelBuilder() { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + // We don't care about the contents since we just want to know that the exception handling works correctly. + return new ByteArraySeekableByteChannel(); + } + }); } @Test @@ -187,7 +183,7 @@ public void testWriteObjectsWithFailedPut() throws SignatureException, IOExcepti final PutObjectResponse putResponse = Mockito.mock(PutObjectResponse.class); Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 0, "foo co"))).thenThrow(new StubException()); Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "baz", jobId, 0, "baz co"))).thenReturn(putResponse); - + final Job job = Ds3ClientHelpers.wrap(ds3Client).startWriteJob(MYBUCKET, Lists.newArrayList( new Ds3Object("foo"), new Ds3Object("bar"), diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java index fc4e83045..eef4c22ae 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/utils/NetUtils_Test.java @@ -15,9 +15,9 @@ package com.spectralogic.ds3client.utils; - import com.spectralogic.ds3client.BulkCommand; import com.spectralogic.ds3client.ConnectionFixture; +import com.spectralogic.ds3client.networking.ConnectionDetails; import com.spectralogic.ds3client.networking.NetUtils; import org.junit.Test; @@ -26,8 +26,9 @@ import java.util.HashMap; import java.util.Map; -import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; public class NetUtils_Test { @@ -143,4 +144,25 @@ public void escapeSpacesInQueryParam() throws MalformedURLException { final URL result = NetUtils.buildUrl(ConnectionFixture.getConnection(), "path", queryParams); assertThat(result.toString(), is("http://localhost:8080/path?foo=bar%20val")); } + + @Test + public void getPortBack() throws MalformedURLException { + final URL url = new URL("http://localhost:8080/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 8080); + } + + @Test + public void getHttpsDefaultPort() throws MalformedURLException { + final URL url = new URL("https://localhost/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 443); + } + + @Test + public void getHttpDefaultPort() throws MalformedURLException { + final URL url = new URL("http://localhost/path"); + int port = NetUtils.getPort(url); + assertTrue(port == 80); + } }