diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientWithCertBasedAuth.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientWithCertBasedAuth.java deleted file mode 100644 index f6e153a37..000000000 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientWithCertBasedAuth.java +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Copyright (c) 2023 MarkLogic Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.marklogic.client.functionaltest; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.DatabaseClientFactory.CertificateAuthContext; -import com.marklogic.client.DatabaseClientFactory.SecurityContext; -import com.marklogic.client.document.TextDocumentManager; -import com.marklogic.client.io.StringHandle; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.util.EntityUtils; -import org.junit.jupiter.api.*; - -import java.io.*; -import java.net.InetAddress; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@Disabled("Ignored because it was previously ignored in build.gradle though without explanation") -public class TestDatabaseClientWithCertBasedAuth extends BasicJavaClientREST { - - public static String newLine = System.getProperty("line.separator"); - public static String temp = System.getProperty("java.io.tmpdir"); - public static String java_home = System.getProperty("java.home"); - public static String server = "CertServer"; - public static String setupServer = "App-Services"; - public static int port = 8071; - public static int setupPort = 8000; - public static String host = "localhost"; - public static DatabaseClient secClient; - public static String localHostname = getBootStrapHostFromML(); - - @BeforeAll - public static void setUpBeforeClass() throws Exception { - - createRESTServerWithDB(server, port); - createRESTUser("portal", "seekrit", "admin", "rest-admin", "rest-writer", "rest-reader"); - associateRESTServerWithDB(setupServer, "Security"); - SecurityContext secContext = newSecurityContext("admin", "admin"); - secClient = DatabaseClientFactory.newClient(host, setupPort, secContext, getConnType()); - - createCACert(); - createCertTemplate(); - createHostTemplate(); - createClientCert("portal"); - convertToHTTPS(); - generateP12("portal"); - - createClientCert("blah"); - generateP12("blah"); - - addCA(); - associateRESTServerWithDB(server, "Documents"); - } - - @AfterAll - public static void tearDownAfterClass() throws Exception { - - removeTrustedCert(); - convertToHTTP(); - removeCertTemplate(); - associateRESTServerWithDB(setupServer, "Documents"); - } - - @BeforeEach - public void setUp() throws Exception { - clearDB(port); - } - - @SuppressWarnings("deprecation") - @Test - public void testUserPortal() throws Exception { - final String query1 = "fn:count(fn:doc())"; - - InetAddress addr = java.net.InetAddress.getLocalHost(); - System.out.println("Hostname is : " + addr.getHostName()); - - DatabaseClient client = DatabaseClientFactory.newClient(localHostname, port, new CertificateAuthContext(temp + "portal.p12", "abc")); - int count = client.newServerEval().xquery(query1).eval().next().getNumber().intValue(); - System.out.println(count); - TextDocumentManager docMgr = client.newTextDocumentManager(); - String docId = "/example/texet.txt"; - StringHandle handle = new StringHandle(); - handle.set("A simple text document"); - docMgr.write(docId, handle); - System.out.println(client.newServerEval().xquery(query1)); - System.out.println("Doc written"); - assertEquals(count + 1, client.newServerEval().xquery(query1).eval().next().getNumber().intValue()); - client.release(); - } - - @SuppressWarnings("deprecation") - @Test - public void testUserBlah() throws Exception { - final String query1 = "fn:count(fn:doc())"; - - DatabaseClient client = DatabaseClientFactory.newClient(localHostname, port, new CertificateAuthContext(temp + "blah.p12", "abc")); - try { - client.newServerEval().xquery(query1).eval().next().getNumber().intValue(); - } catch (Exception e) { - e.printStackTrace(); - System.out.println("Exception is " + e.getClass().getName()); - assertTrue(e.getClass().getName().equals("com.marklogic.client.FailedRequestException")); - assertTrue(e.getMessage().equals("Local message: failed to apply resource at eval: Unauthorized. Server Message: Unauthorized")); - - } - } - - private static void runQuery(String query) throws Exception { - secClient.newServerEval().xquery(query).eval(); - } - - public static void createCACert() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("let $keys := xdmp:rsa-generate()"); - q.append("let $privkey := $keys[1]"); - q.append("let $pubkey := $keys[2]"); - q.append("let $subject :="); - q.append(" element x509:subject {"); - q.append(" element x509:countryName{\"US\"},"); - q.append(" element x509:organizationName{\"Acme Corporation\"},"); - q.append(" element x509:commonName{\"Acme Corporation CA\"}"); - q.append(" }"); - q.append("let $x509 :="); - q.append(" element x509:cert {"); - q.append(" element x509:version {2},"); - q.append(" element x509:serialNumber {pki:integer-to-hex(xdmp:random())},"); - q.append(" element x509:issuer {$subject/*},"); - q.append(" element x509:validity {"); - q.append(" element x509:notBefore {fn:current-dateTime()},"); - q.append(" element x509:notAfter {fn:current-dateTime() + xs:dayTimeDuration(\"P365D\")}"); - q.append(" },"); - q.append("$subject,"); - q.append(" element x509:publicKey{$pubkey},"); - q.append(" element x509:v3ext{"); - q.append(" element x509:basicConstraints {"); - q.append(" attribute critical {\"false\"},"); - q.append(" \"CA:TRUE\""); - q.append(" },"); - q.append(" element x509:keyUsage{"); - q.append(" attribute critical {\"false\"},"); - q.append(" \"Certificate Sign, CRL Sign\""); - q.append(" },"); - q.append(" element x509:nsCertType {"); - q.append(" attribute critical {\"false\"},"); - q.append(" \"SSL Server\""); - q.append(" },"); - q.append(" element x509:subjectKeyIdentifier {"); - q.append(" attribute critical {\"false\"},"); - q.append(" pki:integer-to-hex(xdmp:random())"); - q.append(" }"); - q.append(" }"); - q.append(" }"); - q.append(" let $certificate := xdmp:x509-certificate-generate($x509, $privkey)"); - q.append(" let $dum := xdmp:save(\"" + temp + "ca.cer\", text{$certificate})"); - q.append(" return"); - q.append(" (sec:create-credential("); - q.append(" \"acme-ca\", \"Acme Certificate Authority\","); - q.append(" (),(),$certificate, $privkey,"); - q.append("fn:true(), (), xdmp:permission(\"admin\", \"read\")),"); - q.append("pki:insert-trusted-certificates($certificate)"); - q.append(")"); - System.out.println("Creating CA credential"); - try { - runQuery(q.toString()); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - public static void createCertTemplate() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("pki:insert-template("); - q.append(" pki:create-template("); - q.append(" \"cert-template\", \"testing secure credentials\","); - q.append(" (), (),"); - q.append(" "); - q.append(" 0"); - q.append(" "); - q.append(" US"); - q.append(" CA"); - q.append(" San Carlos"); - q.append(" Acme Corporation"); - q.append(" "); - q.append(" "); - q.append(" SSL Server"); - q.append(" {pki:integer-to-hex(xdmp:random())}"); - q.append(" "); - q.append(" ))"); - System.out.println("Creating Certificate template"); - runQuery(q.toString()); - } - - public static void createHostTemplate() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("let $csr-pem :="); - q.append(" xdmp:invoke-function("); - q.append(" function() {"); - q.append(" pki:generate-certificate-request("); - q.append(" pki:get-template-by-name(\"cert-template\")/pki:template-id,"); - q.append(" xdmp:host-name(), (), ())"); - q.append(" },"); - q.append(" "); - q.append(" update-auto-commit"); - q.append(" different-transaction"); - q.append(" )"); - q.append("let $csr-xml := xdmp:x509-request-extract($csr-pem)"); - q.append("let $ca-xml :="); - q.append(" xdmp:x509-certificate-extract("); - q.append(" xdmp:credential(xdmp:credential-id(\"acme-ca\"))"); - q.append(" /sec:credential-certificate)"); - q.append("let $cert-xml :="); - q.append(" "); - q.append(" 2"); - q.append(" {pki:integer-to-hex(xdmp:random())}"); - q.append(" {$ca-xml/x509:issuer}"); - q.append(" "); - q.append(" {fn:current-dateTime()}"); - q.append(" {fn:current-dateTime() + xs:dayTimeDuration(\"P365D\")}"); - q.append(" "); - q.append(" {$csr-xml/x509:subject}"); - q.append(" {$csr-xml/x509:publicKey}"); - q.append(" {$csr-xml/x509:v3ext}"); - q.append(" "); - q.append("let $cert-pem :="); - q.append(" xdmp:x509-certificate-generate("); - q.append(" $cert-xml, (),"); - q.append(" "); - q.append(" {xdmp:credential-id(\"acme-ca\")}"); - q.append(" )"); - q.append("return"); - q.append(" xdmp:invoke-function("); - q.append(" function() {"); - q.append(" pki:insert-signed-certificates($cert-pem)"); - q.append(" },"); - q.append(" "); - q.append(" update-auto-commit"); - q.append(" different-transaction"); - q.append(" )"); - System.out.println("Creating Host template"); - runQuery(q.toString()); - } - - public static void createClientCert(String commonname) - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module namespace sec = \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("let $validity := element x509:validity {"); - q.append("element x509:notBefore{fn:current-dateTime()},"); - q.append("element x509:notAfter{fn:current-dateTime() + xs:dayTimeDuration(\"P365D\")}}"); - q.append(" let $keys := xdmp:rsa-generate()"); - q.append(" let $privkey := $keys[1]"); - q.append(" let $privkeysave := xdmp:save(\"" + temp + commonname + "priv.pkey\", text{$privkey})"); - q.append(" let $pubkey := $keys[2]"); - q.append(" let $subject :="); - q.append("element x509:subject {"); - q.append("element x509:countryName{\"US\"},"); - q.append("element x509:organizationName{\"Acme Corporation\"},"); - q.append("element x509:commonName{\"" + commonname + "\"}"); - q.append("}"); - q.append(" let $x509 :="); - q.append("element x509:cert {"); - q.append("element x509:version {2},"); - q.append("element x509:serialNumber{pki:integer-to-hex(xdmp:random())},"); - q.append("$validity,"); - q.append("$subject,"); - q.append("element x509:publicKey{$pubkey},"); - q.append("element x509:v3ext {"); - q.append("element x509:subjectKeyIdentifier {"); - q.append("attribute critical {\"false\"},"); - q.append("pki:integer-to-hex(xdmp:random())"); - q.append("}"); - q.append("}"); - q.append("}"); - q.append(" let $certificate := xdmp:x509-certificate-generate($x509, $privkey, {xdmp:credential-id(\"acme-ca\")})"); - q.append(" return xdmp:save(\"" + temp + commonname + ".cer\", text{$certificate})"); - - try { - System.out.println("Creating client credential: " + commonname); - runQuery(q.toString()); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - public static void convertToHTTPS() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module namespace admin = \"http://marklogic.com/xdmp/admin\" at \"/MarkLogic/admin.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("import module namespace sec= \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append(" let $cfg := admin:get-configuration()"); - q.append(" let $group-id := xdmp:group()"); - q.append(" let $app-server-id := admin:appserver-get-id($cfg, $group-id, \"" + server + "\")[1]"); - q.append(" let $template := pki:get-template-by-name(\"cert-template\")"); - q.append(" let $template-id := $template/pki:template-id"); - q.append(" let $client-ca :="); - q.append(" pki:get-certificates(pki:get-trusted-certificate-ids())"); - q.append(" [pki:authority = fn:true()]"); - q.append(" [x509:cert/x509:subject/x509:commonName = \"Acme Corporation CA\"]"); - q.append(" /pki:certificate-id"); - q.append(" let $cfg := admin:appserver-set-authentication($cfg, $app-server-id, \"certificate\")"); - q.append(" let $cfg := admin:appserver-set-ssl-certificate-template($cfg, $app-server-id, $template-id)"); - q.append(" let $cfg := admin:appserver-set-ssl-client-certificate-authorities($cfg, $app-server-id, $client-ca)"); - q.append(" let $cfg := admin:appserver-set-ssl-require-client-certificate($cfg, $app-server-id, fn:true())"); - q.append("return admin:save-configuration($cfg)"); - runQuery(q.toString()); - } - - private static void addCA() throws IOException, InterruptedException { - - String seperator = File.separator; - System.out.println(java_home + seperator + "lib" + seperator + "security" + seperator + "cacerts"); - System.out.println(temp + "ca.cer"); - - Runtime rt = Runtime.getRuntime(); - Process pr = rt.exec(new String[] { "keytool", "-import", "-trustcacerts", "-noprompt", "-storepass", "changeit", "-file", temp + "ca.cer", "-alias", "Acme", "-keystore", - "\"" + java_home + seperator + "lib" + seperator + "security" + seperator + "cacerts" + "\"" }); - System.out.println(pr.waitFor()); - InputStream is = pr.getInputStream(); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader buff = new BufferedReader(isr); - - String line; - System.out.println("Adding CA to trusted certificate: STDOUT"); - while ((line = buff.readLine()) != null) - System.out.println(line); - - InputStream is1 = pr.getErrorStream(); - InputStreamReader isr1 = new InputStreamReader(is1); - BufferedReader buff1 = new BufferedReader(isr1); - - String line1; - System.out.println("Adding CA to trusted certificate: ERR"); - while ((line1 = buff1.readLine()) != null) - System.out.println(line1); - Thread.currentThread().sleep(5000L); - - } - - private static void generateP12(String commonname) throws IOException, InterruptedException { - Runtime rt = Runtime.getRuntime(); - Process pr = rt.exec(new String[] { "openssl", "pkcs12", "-export", "-in", temp + commonname + ".cer", "-inkey", temp + commonname + "priv.pkey", "-out", - temp + commonname + ".p12", "-passout", "pass:abc" }); - System.out.println(pr.waitFor()); - InputStream is = pr.getInputStream(); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader buff = new BufferedReader(isr); - - String line; - while ((line = buff.readLine()) != null) - System.out.println(line); - - InputStream is1 = pr.getErrorStream(); - InputStreamReader isr1 = new InputStreamReader(is1); - BufferedReader buff1 = new BufferedReader(isr1); - - String line1; - while ((line1 = buff1.readLine()) != null) - System.out.println(line1); - Thread.currentThread().sleep(5000L); - - } - - public static void convertToHTTP() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("import module namespace sec= \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("import module namespace admin= \"http://marklogic.com/xdmp/admin\" at \"/MarkLogic/admin.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("let $cfg := admin:get-configuration()"); - q.append("let $group-id := xdmp:group()"); - q.append("let $app-server-id := admin:appserver-get-id($cfg, $group-id, \"" + server + "\")[1]"); - q.append("let $cfg := admin:appserver-set-authentication($cfg, $app-server-id, \"digest\")"); - q.append("let $cfg := admin:appserver-set-ssl-certificate-template($cfg, $app-server-id, 0)"); - q.append("let $cfg := admin:appserver-set-ssl-client-certificate-authorities($cfg, $app-server-id, ())"); - q.append("return admin:save-configuration($cfg)"); - runQuery(q.toString()); - } - - public static void removeCertTemplate() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("pki:delete-template(pki:get-template-by-name(\"cert-template\")/pki:template-id)"); - System.out.println("Removing Certificate Template"); - runQuery(q.toString()); - } - - public static void removeTrustedCert() - throws Exception { - String seperator = File.separator; - Runtime rt = Runtime.getRuntime(); - Process pr = rt.exec(new String[] { "keytool", "-delete", "-noprompt", "-storepass", "changeit", "-alias", "Acme", "-keystore", - "\"" + java_home + seperator + "lib" + seperator + "security" + seperator + "cacerts" + "\"" }); - System.out.println(pr.waitFor()); - - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module namespace admin = \"http://marklogic.com/xdmp/admin\" at \"/MarkLogic/admin.xqy\";"); - q.append("import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";"); - q.append("import module namespace sec= \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("declare namespace x509 = \"http://marklogic.com/xdmp/x509\";"); - q.append("( \"acme-ca\" ) ! sec:remove-credential(.), "); - q.append("pki:delete-certificate(pki:get-certificates(pki:get-trusted-certificate-ids())[pki:authority = fn:true()][x509:cert/x509:subject/x509:commonName = (\"Acme Corporation CA\")]/pki:certificate-id/text())"); - System.out.println("Removing from Trusted Certificates"); - runQuery(q.toString()); - } - - public static void removeCredentials() - throws Exception { - StringBuilder q = new StringBuilder(); - q.append("xquery version \"1.0-ml\";"); - q.append("import module \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\";"); - q.append("xdmp:credentials()//sec:credential-name/text()! sec:remove-credential(.)"); - runQuery(q.toString()); - } - - public static void clearDB(int port) { - DefaultHttpClient client = null; - try { - InputStream jsonstream = null; - // In case of SSL use 8002 port to clear DB contents. - client = new DefaultHttpClient(); - client.getCredentialsProvider().setCredentials( - new AuthScope(host, 8002), - new UsernamePasswordCredentials("admin", "admin")); - HttpGet getrequest = new HttpGet("http://" + host + ":8002/manage/v2/servers/" + server + "/properties?group-id=Default&format=json"); - HttpResponse response1 = client.execute(getrequest); - jsonstream = response1.getEntity().getContent(); - JsonNode jnode = new ObjectMapper().readTree(jsonstream); - String dbName = jnode.get("content-database").asText(); - System.out.println("App Server's content database properties value from ClearDB is :" + dbName); - - ObjectMapper mapper = new ObjectMapper(); - ObjectNode mainNode = mapper.createObjectNode(); - - mainNode.put("operation", "clear-database"); - - HttpPost post = new HttpPost("http://" + host + ":8002" + "/manage/v2/databases/" + dbName); - post.addHeader("Content-type", "application/json"); - post.setEntity(new StringEntity(mainNode.toString())); - - HttpResponse response = client.execute(post); - HttpEntity respEntity = response.getEntity(); - if (response.getStatusLine().getStatusCode() == 400) { - System.out.println("Database contents cleared"); - } - else if (respEntity != null) { - // EntityUtils to get the response content - String content = EntityUtils.toString(respEntity); - System.out.println(content); - } - else { - System.out.println("No Proper Response from clearDB in SSL."); - } - - } catch (Exception e) { - // writing error to Log - e.printStackTrace(); - } finally { - client.getConnectionManager().shutdown(); - } - } - -} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index d2b2f93e8..d114b2ea5 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -185,6 +185,14 @@ private FailedRequest extractErrorFields(Response response) { failure.setStatusString("Forbidden"); failure.setStatusCode(STATUS_FORBIDDEN); return failure; + } else if (response.code() == STATUS_FORBIDDEN && "".equals(responseBody)) { + // When the responseBody is empty, this seems preferable vs the "Server (not a REST instance?)" message + // which is very confusing given that the app server usually is a REST instance. + FailedRequest failure = new FailedRequest(); + failure.setMessageString("No message received from server."); + failure.setStatusString("Forbidden"); + failure.setStatusCode(STATUS_FORBIDDEN); + return failure; } InputStream is = new ByteArrayInputStream(responseBody.getBytes(StandardCharsets.UTF_8)); @@ -515,6 +523,13 @@ private Response sendRequestOnce(Request request) { try { return getConnection().newCall(request).execute(); } catch (IOException e) { + if (e instanceof SSLException) { + String message = e.getMessage(); + if (message != null && message.contains("readHandshakeRecord")) { + throw new MarkLogicIOException(String.format("SSL error occurred: %s; ensure you are using a valid certificate " + + "if the MarkLogic app server requires a client certificate for SSL.", message)); + } + } String message = String.format( "Error occurred while calling %s; %s: %s " + "; possible reasons for the error include that a MarkLogic app server may not be listening on the port, " + diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/TwoWaySSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/TwoWaySSLTest.java new file mode 100644 index 000000000..fb098c4c2 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/TwoWaySSLTest.java @@ -0,0 +1,360 @@ +package com.marklogic.client.test; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.ForbiddenUserException; +import com.marklogic.client.MarkLogicIOException; +import com.marklogic.client.document.DocumentDescriptor; +import com.marklogic.client.eval.EvalResultIterator; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.junit5.RequireSSLExtension; +import com.marklogic.mgmt.ManageClient; +import com.marklogic.mgmt.resource.appservers.ServerManager; +import com.marklogic.rest.util.Fragment; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.util.FileCopyUtils; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(RequireSSLExtension.class) +public class TwoWaySSLTest { + + private final static String TEST_DOCUMENT_URI = "/optic/test/musician1.json"; + + // Used for creating a temporary JKS (Java KeyStore) file. + @TempDir + static Path tempDir; + + private static DatabaseClient securityClient; + private static ManageClient manageClient; + private static File keyStoreFile; + + + @BeforeAll + public static void setup() throws Exception { + // Create a client using the java-unittest app server - which requires SSL via RequiresSSLExtension - and that + // talks to the Security database. + securityClient = Common.newClientBuilder() + .withUsername(Common.SERVER_ADMIN_USER).withPassword(Common.SERVER_ADMIN_PASS) + .withSSLProtocol("TLSv1.2") + .withTrustManager(Common.TRUST_ALL_MANAGER) + .withSSLHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY) + .withDatabase("Security").build(); + manageClient = Common.newManageClient(); + + final String certificateAuthorityId = createCertificateAuthority(); + ClientCertificate clientCertificate = createClientCertificate(); + makeAppServerRequireTwoWaySSL(certificateAuthorityId); + + writeClientCertificateFilesToTempDir(clientCertificate, tempDir); + createPkcs12File(tempDir); + createKeystoreFile(tempDir); + keyStoreFile = new File(tempDir.toFile(), "client.jks"); + } + + @AfterAll + public static void teardown() { + removeTwoWaySSLConfig(); + deleteCertificateAuthority(); + } + + /** + * After two-way SSL is configured on the java-unittest app server, verify that a DatabaseClient using a proper + * SSLContext can connect to the app server. + */ + @Test + void digestAuthentication() throws Exception { + if (Common.USE_REVERSE_PROXY_SERVER) { + return; + } + + // This client uses our Java KeyStore file with a client certificate in it, so it should work. + DatabaseClient clientWithCert = Common.newClientBuilder() + .withSSLHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY) + .withSSLContext(createSSLContextWithClientCertificate(keyStoreFile)) + .withTrustManager(RequireSSLExtension.newTrustManager()) + .build(); + + verifyTestDocumentCanBeRead(clientWithCert); + + // This client uses a new SSL context without the client certificate, so it should fail. + DatabaseClient clientWithoutCert = Common.newClientBuilder() + .withSSLHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY) + .withSSLProtocol("TLSv1.2") + .withTrustManager(RequireSSLExtension.newTrustManager()) + .build(); + + // The type of SSL failure varies across Java versions, so not asserting on a particular error message. + assertThrows(MarkLogicIOException.class, + () -> clientWithoutCert.newJSONDocumentManager().exists(TEST_DOCUMENT_URI)); + + // And now a client that doesn't even try to use SSL. It's not clear if a ForbiddenUserException is correct + // here, but it's what the Java Client was throwing when this test was written. + ForbiddenUserException userException = assertThrows(ForbiddenUserException.class, + () -> Common.newClient().newJSONDocumentManager().exists(TEST_DOCUMENT_URI)); + assertTrue(userException.getMessage().contains("User is not allowed to check the existence of documents"), + "Unexpected exception: " + userException.getMessage()); + } + + @Test + void certificateAuthentication() throws Exception { + if (Common.USE_REVERSE_PROXY_SERVER) { + return; + } + + try { + new ServerManager(manageClient) + .save(Common.newServerPayload().put("authentication", "certificate").toString()); + + SSLContext sslContext = createSSLContextWithClientCertificate(keyStoreFile); + DatabaseClient client = Common.newClientBuilder() + .withCertificateAuth(sslContext, RequireSSLExtension.newTrustManager()) + .withSSLHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY) + .build(); + + verifyTestDocumentCanBeRead(client); + } finally { + new ServerManager(manageClient) + .save(Common.newServerPayload().put("authentication", "digestbasic").toString()); + } + } + + private void verifyTestDocumentCanBeRead(DatabaseClient client) { + DocumentDescriptor descriptor = client.newJSONDocumentManager().exists(TEST_DOCUMENT_URI); + assertNotNull(descriptor); + assertEquals(TEST_DOCUMENT_URI, descriptor.getUri()); + } + + private SSLContext createSSLContextWithClientCertificate(File keystoreFile) throws Exception { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(new FileInputStream(keystoreFile), "password".toCharArray()); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("SunX509"); + keyManagerFactory.init(keyStore, "password".toCharArray()); + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init( + keyManagerFactory.getKeyManagers(), + new X509TrustManager[]{RequireSSLExtension.newTrustManager()}, + null); + return sslContext; + } + + /** + * See https://docs.marklogic.com/pki:create-authority for more information. This results in both a new + * CA in MarkLogic and a new "secure credential". + */ + private static String createCertificateAuthority() { + String xquery = "xquery version \"1.0-ml\";\n" + + "import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";\n" + + "declare namespace x509 = \"http://marklogic.com/xdmp/x509\";\n" + + "\n" + + "pki:create-authority(\n" + + " \"java-client-test\", \"Java Client Certificate Authority\",\n" + + " element x509:subject {\n" + + " element x509:countryName {\"US\"},\n" + + " element x509:stateOrProvinceName {\"California\"},\n" + + " element x509:localityName {\"San Carlos\"},\n" + + " element x509:organizationName {\"MarkLogicJavaClientTest\"},\n" + + " element x509:organizationalUnitName {\"Engineering\"},\n" + + " element x509:commonName {\"JavaClientCA\"},\n" + + " element x509:emailAddress {\"java-client@example.org\"}\n" + + " },\n" + + " fn:current-dateTime(),\n" + + " fn:current-dateTime() + xs:dayTimeDuration(\"P365D\"),\n" + + " (xdmp:permission(\"admin\",\"read\")))"; + + return securityClient.newServerEval().xquery(xquery).evalAs(String.class); + } + + /** + * See https://docs.marklogic.com/pki:authority-create-client-certificate for more information. + * The commonName matches that of a known test user so that certificate authentication can be tested too. + */ + private static ClientCertificate createClientCertificate() { + String xquery = "xquery version \"1.0-ml\";\n" + + "import module namespace sec = \"http://marklogic.com/xdmp/security\" at \"/MarkLogic/security.xqy\"; \n" + + "import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";\n" + + "declare namespace x509 = \"http://marklogic.com/xdmp/x509\";\n" + + "\n" + + "pki:authority-create-client-certificate(\n" + + " xdmp:credential-id(\"java-client-test\"),\n" + + " element x509:subject {\n" + + " element x509:countryName {\"US\"},\n" + + " element x509:stateOrProvinceName {\"California\"},\n" + + " element x509:localityName {\"San Carlos\"},\n" + + " element x509:organizationName {\"ProgressMarkLogic\"},\n" + + " element x509:organizationalUnitName {\"Engineering\"},\n" + + " element x509:commonName {\"JavaClientCertificateUser\"},\n" + + " element x509:emailAddress {\"java.client@example.org\"}\n" + + " },\n" + + " fn:current-dateTime(),\n" + + " fn:current-dateTime() + xs:dayTimeDuration(\"P365D\"))\n"; + + EvalResultIterator iter = securityClient.newServerEval().xquery(xquery).eval(); + String cert = null; + String key = null; + while (iter.hasNext()) { + if (cert == null) { + cert = iter.next().getString(); + } else { + key = iter.next().getString(); + } + } + return new ClientCertificate(cert, key); + } + + private static class ClientCertificate { + final String pemEncodedCertificate; + final String privateKey; + + public ClientCertificate(String pemEncodedCertificate, String privateKey) { + this.pemEncodedCertificate = pemEncodedCertificate; + this.privateKey = privateKey; + } + } + + /** + * Via the RequiresSSLExtension, the app server already requires a 1-way SSL connection. This configures the + * app server to both require a client certificate and have that client certificate associated with the + * CA that was created earlier in the test. + */ + private static void makeAppServerRequireTwoWaySSL(String certificateAuthorityId) { + String certificateAuthorityCertificate = getCertificateAuthorityCertificate(certificateAuthorityId); + ObjectNode payload = Common.newServerPayload() + .put("ssl-require-client-certificate", true) + .put("ssl-client-issuer-authority-verification", true); + payload.putArray("ssl-client-certificate-pem").add(certificateAuthorityCertificate); + new ServerManager(manageClient).save(payload.toString()); + } + + /** + * Couldn't find a Manage API endpoint that returns the CA certificate, so directly accessing the Security + * database and reading a known URI to get an XML document associated with the CA's secure credential, which has + * the certificate in it. + */ + private static String getCertificateAuthorityCertificate(String certificateAuthorityId) { + String certificateUri = String.format("http://marklogic.com/xdmp/credentials/%s", certificateAuthorityId); + String xml = securityClient.newXMLDocumentManager().read(certificateUri, new StringHandle()).get(); + return new Fragment(xml).getElementValue("/sec:credential/sec:credential-certificate"); + } + + /** + * Note that if this test fails and the CA somehow doesn't get deleted, you can delete it manually via the Admin + * UI - but you need to delete two things - the CA, and then there's a "java-client-test" Secure Credential that + * needs to be deleted as well. + */ + private static void deleteCertificateAuthority() { + String xquery = "xquery version \"1.0-ml\";\n" + + "import module namespace pki = \"http://marklogic.com/xdmp/pki\" at \"/MarkLogic/pki.xqy\";\n" + + "\n" + + "pki:delete-authority(\"java-client-test\")"; + + securityClient.newServerEval().xquery(xquery).evalAs(String.class); + } + + /** + * Restores the app server back to only requiring 1-way SSL. + */ + private static void removeTwoWaySSLConfig() { + ObjectNode payload = Common.newServerPayload() + .put("ssl-require-client-certificate", false) + .put("ssl-client-issuer-authority-verification", false); + payload.putArray("ssl-client-certificate-pem"); + new ServerManager(manageClient).save(payload.toString()); + } + + /** + * Writes the client certificate PEM and private keys to disk so that they can accessed by the openssl program. + * + * @param clientCertificate + * @param tempDir + * @throws IOException + */ + private static void writeClientCertificateFilesToTempDir(ClientCertificate clientCertificate, Path tempDir) throws IOException { + File certFile = new File(tempDir.toFile(), "cert.pem"); + FileCopyUtils.copy(clientCertificate.pemEncodedCertificate.getBytes(), certFile); + File keyFile = new File(tempDir.toFile(), "client.key"); + FileCopyUtils.copy(clientCertificate.privateKey.getBytes(), keyFile); + } + + /** + * See https://stackoverflow.com/a/8224863/3306099 for where this approach was obtained from. + */ + private static void createPkcs12File(Path tempDir) throws Exception { + ProcessBuilder builder = new ProcessBuilder(); + builder.directory(tempDir.toFile()); + builder.command("openssl", "pkcs12", "-export", + "-in", "cert.pem", "-inkey", "client.key", + "-out", "client.p12", + "-name", "my-client", + "-passout", "pass:password"); + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Process process = builder.start(); + executorService.submit(new StreamGobbler(process.getInputStream(), System.out::println)); + executorService.submit(new StreamGobbler(process.getErrorStream(), System.err::println)); + int exitCode = process.waitFor(); + assertEquals(0, exitCode, "Unable to create pkcs12 file using openssl"); + } + + private static void createKeystoreFile(Path tempDir) throws Exception { + ProcessBuilder builder = new ProcessBuilder(); + builder.directory(tempDir.toFile()); + builder.command("keytool", "-importkeystore", + "-deststorepass", "password", + "-destkeypass", "password", + "-destkeystore", "client.jks", + "-srckeystore", "client.p12", + "-srcstoretype", "PKCS12", + "-srcstorepass", "password", + "-alias", "my-client"); + + Process process = builder.start(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.submit(new StreamGobbler(process.getInputStream(), System.out::println)); + executorService.submit(new StreamGobbler(process.getErrorStream(), System.err::println)); + int exitCode = process.waitFor(); + assertEquals(0, exitCode, "Unable to create keystore using keytool"); + } + + /** + * Copied from https://www.baeldung.com/run-shell-command-in-java . + */ + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } +} diff --git a/test-app/src/main/ml-config/security/users/certificate-user.json b/test-app/src/main/ml-config/security/users/certificate-user.json new file mode 100644 index 000000000..d793dc30e --- /dev/null +++ b/test-app/src/main/ml-config/security/users/certificate-user.json @@ -0,0 +1,8 @@ +{ + "user-name": "JavaClientCertificateUser", + "description": "java-client-api user for testing certificate authentication.", + "password": "doesnt-matter-wont-be-used", + "role": [ + "rest-reader" + ] +}