[Backport 2.6][#8789] Add support for certificate chaining in platfor…

…m/yb-client. Summary: The platform worked under the assumption that each certificate input will be a single certificate object, which is an incorrect assumption in most situations, since this will not be the case in most situations. We will almost always have a certificate chain with an intermediate and a root. This diff ensures we respect all certificates being sent as input while uploading the certificate content, as well as it ensures that the yb-client uses all certs present in the file for the trust store. Original diff: https://phabricator.dev.yugabyte.com/D11839 Original commit: 930d103 Test Plan: Jenkins: rebase: 2.6 yb-client: Added a unit test. platform: 1) Added a unit test. 2) Verified via the cloud workflow. Sent a chain of self-signed certificate to use for generating the server certificates, and verified the universe creation worked as expected. Reviewers: sanketh, arnav Reviewed By: arnav Subscribers: yugaware, asingh, jenkins-bot Differential Revision: https://phabricator.dev.yugabyte.com/D12384
yugabyte · Jul 27, 2021 · 4bd9b02 · 4bd9b02
1 parent 978cb20
commit 4bd9b02
Show file tree

Hide file tree

Showing 6 changed files with 391 additions and 239 deletions.
diff --git a/ent/test_certs/multiCA.crt b/ent/test_certs/multiCA.crt
@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIDAjCCAeqgAwIBAgIGAWrm7jKOMA0GCSqGSIb3DQEBCwUAMC4xFjAUBgNVBAMM
+DXliLWFkbWluLXRlc3QxFDASBgNVBAoMC2V4YW1wbGUuY29tMB4XDTE5MDUyMzIz
+MDIxMVoXDTIwMDUyMzIzMDIxMVowLjEWMBQGA1UEAwwNeWItYWRtaW4tdGVzdDEU
+MBIGA1UECgwLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCsiEmshRP3c/2XCjhcp2JUx+C3S/PZTczhbw284Nqp911WGdHfU1AYroOl
+aIH4/T3LSaY6QZJCekWzRur0zrSU9bM3OqBQTGaZ/LBkrVDY/3XoPWikf9wIHFcK
+sX7b13/nbjEUDYl19Vmzukrxag7RVRFuRegrM119i7riXKUHiio/645MGxwKmRSu
+1A95985GdYPPhKl3XzeEEHmp6pmorFeJciqCOotJA1LWpGRCAfoYZV9MNkFqZO0q
+AyjxnQS69GNALvT8jehXclUfcdFtyaWQqLZmwY520dE2txz3GSTQb5DLLQwKMZYc
+lbTMy2l83cB4e+tGQudqQ7LpBiivAgMBAAGjJjAkMBIGA1UdEwEB/wQIMAYBAf8C
+AQEwDgYDVR0PAQH/BAQDAgLkMA0GCSqGSIb3DQEBCwUAA4IBAQAgCV3Q+uMAo5n/
+QPjjd6q6YzAYvoKHqKYKLSh/JcOg22zu7o3VUW0pLzRVtXeRdVefSKURhvZjHMnP
+1czh2xVpsUwmA/L5/2SiIO1nVWFYe8TyRwP03Fvrr4mKQVlWDSAL5IQ5fxGPRcQa
+HcpiD4/Owt9LMVdm7csnMiRbREg+soajnjnISJxX37IWlDbZMsvIQQK9cWwMklQv
+aZc3yaI82i/4NFgLq3Epl5rGphHPODp7RupuRWQsGxHqBzv51ip5X6qpoQ8uhLHl
+RCGnOCIGkxwt5dZQJne+rajlHtTVr8j/4hNiN2hohbKLwzk+4VJlTtc3DuLR6Eg7
+Y+IbydAD
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICqDCCAZACCQCZzuWUK8cZ7jANBgkqhkiG9w0BAQsFADAWMRQwEgYDVQQDDAtZ
+dWdhQnl0ZSBDQTAeFw0xODA2MDUyMjA5MTZaFw0yODA2MTIyMjA5MTZaMBYxFDAS
+BgNVBAMMC1l1Z2FCeXRlIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEArmefuP3J1JBojKc710ATZaDnjDnIRjcRR4Af4jj2k8zZhw/5DoLPSmGCmTSq
+t1ZvFe9a2TAAAEZLceV+6ysH+4G2E8haVCYPDlvz00Wt1NFhCUg/zEFpMxDT84HC
+HSXPaZKjXR76lp3Afc/xMgK7Q5ZlOaTJ/kJVgVb/SLgq1DjuvQkqQLbQjZLa6vGf
+kpU037dJUaptjIPTnmX9OOrOdDkF0lNzMbnL5gwWWIpJofBKT+d4u/d7oX00AC9H
+q9oautJOUwNymZkgla12pOlhGOnXJ6tqhoY7EfKM2fpzAGIwAe69a8aIv8zWBBfQ
+f8ASQT5TZa4JPrZiBNITPGjHrwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCZN1mL
+0jdSLkLh0xaX4Z6icrz+8Hd2dn4Qg1VQtBtW2uYh2ds3SBHQqnoos+tdTJ4WzC7J
+hSuIp0d6JaA4ZS7BO8ucH0UzPh0SFfx9NWNiv6aA8SJta0HLD6DBSftdU+YlLWbH
+t7D96rOpdf+mJCTpHcJ2HLQWFF1i6cJe1rFhemb0+VHAV1JIC5C5XiJYZmlOYDeG
++IgScyuPmD0fmVdccofm1XxdEdpT2Y/PVZvbfqOfPBpXr6+kU1zjKURfgMyap6nv
+JuIho07OU6iGgb41XoQLjC1Ft7AdgPqqCQ141ROpC4Nyne5RDvvCqLiHKJEWpmGo
+tyT7FPRPXZhcROGq
+-----END CERTIFICATE-----
diff --git a/java/yb-client/src/main/java/org/yb/client/AsyncYBClient.java b/java/yb-client/src/main/java/org/yb/client/AsyncYBClient.java
@@ -51,67 +51,25 @@
 import com.google.protobuf.Message;
 import com.stumbleupon.async.Callback;
 import com.stumbleupon.async.Deferred;
-
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemReader;
-import org.yb.Common;
-import org.yb.Common.YQLDatabase;
-import org.yb.Schema;
-import org.yb.annotations.InterfaceAudience;
-import org.yb.annotations.InterfaceStability;
-import org.yb.consensus.Metadata;
-import org.yb.master.Master;
-import org.yb.master.Master.GetTableLocationsResponsePB;
-import org.yb.master.Master.ListTablesResponsePB.TableInfo;
-import org.yb.util.AsyncUtil;
-import org.yb.util.NetUtil;
-import org.yb.util.Pair;
-import org.yb.util.Slice;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.ChannelEvent;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.DefaultChannelPipeline;
-import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
-import org.jboss.netty.channel.socket.SocketChannel;
-import org.jboss.netty.channel.socket.SocketChannelConfig;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
-import org.jboss.netty.handler.ssl.SslHandler;
-import org.jboss.netty.util.HashedWheelTimer;
-import org.jboss.netty.util.Timeout;
-import org.jboss.netty.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.TrustManagerFactory;
-
-import java.security.cert.CertificateFactory;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-import java.security.KeyStore;
-import java.security.KeyFactory;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-
 import java.io.FileInputStream;
 import java.io.FileReader;
-
-import javax.annotation.concurrent.GuardedBy;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.net.UnknownHostException;
 import java.nio.charset.Charset;
+import java.security.KeyFactory;
+import java.security.KeyStore;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -122,7 +80,40 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
-
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.TrustManagerFactory;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.ChannelEvent;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.DefaultChannelPipeline;
+import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
+import org.jboss.netty.channel.socket.SocketChannel;
+import org.jboss.netty.channel.socket.SocketChannelConfig;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.ssl.SslHandler;
+import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yb.Common;
+import org.yb.Common.YQLDatabase;
+import org.yb.Schema;
+import org.yb.annotations.InterfaceAudience;
+import org.yb.annotations.InterfaceStability;
+import org.yb.consensus.Metadata;
+import org.yb.master.Master;
+import org.yb.master.Master.GetTableLocationsResponsePB;
+import org.yb.util.AsyncUtil;
+import org.yb.util.NetUtil;
+import org.yb.util.Pair;
+import org.yb.util.Slice;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 /**
@@ -2115,6 +2106,7 @@ private PrivateKey getPrivateKey(String keyFile) {
       try {
         PemReader pemReader = new PemReader(new FileReader(keyFile));
         PemObject pemObject = pemReader.readPemObject();
+        pemReader.close();
         byte[] bytes = pemObject.getContent();
         PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(bytes);
         KeyFactory kf = KeyFactory.getInstance("RSA");
@@ -2129,15 +2121,16 @@ private PrivateKey getPrivateKey(String keyFile) {
       }
     }
 
+    @SuppressWarnings("unchecked")
     private SslHandler createSslHandler(String certfile, String clientCertFile,
                                         String clientKeyFile) {
       try {
         Security.addProvider(new BouncyCastleProvider());
         CertificateFactory cf = CertificateFactory.getInstance("X.509");
         FileInputStream fis = new FileInputStream(certFile);
-        X509Certificate ca;
+        List<X509Certificate> cas;
         try {
-          ca = (X509Certificate) cf.generateCertificate(fis);
+          cas = (List<X509Certificate>) (List<?>) cf.generateCertificates(fis);
         } catch (Exception e) {
           log.error("Exception generating CA certificate from input file: ", e);
           throw e;
@@ -2149,14 +2142,18 @@ private SslHandler createSslHandler(String certfile, String clientCertFile,
         String keyStoreType = KeyStore.getDefaultType();
         KeyStore keyStore = KeyStore.getInstance(keyStoreType);
         keyStore.load(null, null);
-        keyStore.setCertificateEntry("ca", ca);
+        for (int i = 0; i < cas.size(); i++) {
+          // Adding to the trust store. Expect the caller to have verified
+          // the certs.
+          keyStore.setCertificateEntry("ca_" + i, cas.get(i));
+        }
 
         // Create a TrustManager that trusts the CAs in our KeyStore
         String tmfAlgorithm = TrustManagerFactory.getDefaultAlgorithm();
         TrustManagerFactory tmf = TrustManagerFactory.getInstance(tmfAlgorithm);
         tmf.init(keyStore);
 
-        X509Certificate clientCert = null;
+        List<X509Certificate> clientCerts = null;
         KeyStore clientKeyStore = null;
         KeyManagerFactory kmf = null;
         if (clientCertFile != null) {
@@ -2166,21 +2163,22 @@ private SslHandler createSslHandler(String certfile, String clientCertFile,
           }
           fis = new FileInputStream(clientCertFile);
           try {
-            clientCert = (X509Certificate) cf.generateCertificate(fis);
+            clientCerts = (List<X509Certificate>) (List<?>) cf.generateCertificates(fis);
           } catch (Exception e) {
             log.error("Exception generating CA certificate from input file: ", e);
             throw e;
           } finally {
             fis.close();
           }
           PrivateKey pk = getPrivateKey(clientKeyFile);
-          Certificate[] chain = new Certificate[2];
-          chain[0] = clientCert;
-          chain[1] = ca;
-
+          Certificate[] chain = new Certificate[clientCerts.size()];
           clientKeyStore = KeyStore.getInstance(keyStoreType);
           clientKeyStore.load(null, null);
-          clientKeyStore.setCertificateEntry("node_crt", clientCert);
+          for (int i = 0; i < clientCerts.size(); i++) {
+            chain[i] = clientCerts.get(i);
+            clientKeyStore.setCertificateEntry("node_crt_" + i, clientCerts.get(i));
+          }
+
           String password = "password";
           char[] ksPass = password.toCharArray();
           clientKeyStore.setKeyEntry("node_key", pk, ksPass, chain);

diff --git a/java/yb-client/src/test/java/org/yb/client/TestYBClient.java b/java/yb-client/src/test/java/org/yb/client/TestYBClient.java
@@ -31,37 +31,34 @@
 //
 package org.yb.client;
 
-import static org.yb.AssertionWrappers.assertEquals;
-import static org.yb.AssertionWrappers.assertFalse;
-import static org.yb.AssertionWrappers.assertNotEquals;
-import static org.yb.AssertionWrappers.assertNotNull;
-import static org.yb.AssertionWrappers.assertTrue;
-
-import java.util.*;
-
+import com.google.common.net.HostAndPort;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yb.ColumnSchema;
 import org.yb.Common.HostPortPB;
 import org.yb.Common.TableType;
 import org.yb.Schema;
 import org.yb.Type;
-import org.yb.util.Pair;
-import org.yb.master.Master;
-import org.yb.minicluster.MiniYBCluster;
-import org.yb.tserver.Tserver.TabletServerErrorPB;
-
-import com.google.common.net.HostAndPort;
-
-import com.google.protobuf.ByteString;
-
 import org.yb.YBTestRunner;
-
-import org.junit.runner.RunWith;
+import org.yb.tserver.Tserver.TabletServerErrorPB;
+import org.yb.util.Pair;
 import org.yb.util.Timeouts;
+import static org.yb.AssertionWrappers.assertEquals;
+import static org.yb.AssertionWrappers.assertFalse;
+import static org.yb.AssertionWrappers.assertNotEquals;
+import static org.yb.AssertionWrappers.assertNotNull;
+import static org.yb.AssertionWrappers.assertTrue;
 
 @RunWith(value=YBTestRunner.class)
 public class TestYBClient extends BaseYBClientTest {
@@ -463,7 +460,7 @@ public void testAffinitizedLeaders() throws Exception {
     List<ColumnSchema> columns = new ArrayList<>(hashKeySchema.getColumns());
     Schema newSchema = new Schema(columns);
     CreateTableOptions tableOptions = new CreateTableOptions().setNumTablets(8);
-    YBTable table = syncClient.createTable(DEFAULT_KEYSPACE_NAME, "AffinitizedLeaders", newSchema, tableOptions);
+    syncClient.createTable(DEFAULT_KEYSPACE_NAME, "AffinitizedLeaders", newSchema, tableOptions);
 
     assertTrue(syncClient.waitForAreLeadersOnPreferredOnlyCondition(DEFAULT_TIMEOUT_MS));
 
@@ -577,7 +574,7 @@ public void testGetTablesList() throws Exception {
 
     // Check that YEDIS tables are created and retrieved properly.
     String redisTableName = YBClient.REDIS_DEFAULT_TABLE_NAME;
-    YBTable table = syncClient.createRedisTable(redisTableName);
+    syncClient.createRedisTable(redisTableName);
     assertFalse(syncClient.getTablesList().getTablesList().isEmpty());
     assertTrue(syncClient.getTablesList().getTablesList().contains(redisTableName));
 

diff --git a/java/yb-client/src/test/java/org/yb/clientent/TestSSLClient.java b/java/yb-client/src/test/java/org/yb/clientent/TestSSLClient.java
@@ -12,39 +12,22 @@
 //
 package org.yb.clientent;
 
-import org.yb.client.*;
-
-import java.util.*;
-
-import com.google.protobuf.ByteString;
-
-import org.junit.Ignore;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.util.Map;
+import java.util.TreeMap;
 import org.junit.Test;
-import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
-
-import org.yb.Schema;
-import org.yb.ColumnSchema;
 import org.yb.YBTestRunner;
-import org.yb.master.Master;
-import org.yb.minicluster.MiniYBCluster;
-
+import org.yb.client.AsyncYBClient;
+import org.yb.client.TestUtils;
+import org.yb.client.TestYBClient;
+import org.yb.client.YBClient;
 import org.yb.util.Timeouts;
-
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-
-import static org.yb.AssertionWrappers.assertTrue;
 import static org.yb.AssertionWrappers.assertFalse;
 
 @RunWith(value=YBTestRunner.class)
 public class TestSSLClient extends TestYBClient {
-  private static final String PLACEMENT_CLOUD = "testCloud";
-  private static final String PLACEMENT_REGION = "testRegion";
-  private static final String PLACEMENT_ZONE = "testZone";
-  private static final String LIVE_TS = "live";
-  private static final String READ_ONLY_TS = "readOnly";
-  private static final String READ_ONLY_NEW_TS = "readOnlyNew";
 
   public enum TestMode {
     TLS, MUTUAL_TLS_NO_CLIENT_VERIFY, MUTUAL_TLS_CLIENT_VERIFY
@@ -98,6 +81,32 @@ public void testClientCorrectCertificate() throws Exception {
     myClient = null;
   }
 
+  /**
+   * Test to check that client connection succeeds when provided a file with
+   * multiple root certs.
+   * @throws Exception
+   */
+  @Test(timeout = 100000)
+  public void testClientMultiCertificate() throws Exception {
+    LOG.info("Starting testClientMultiCertificate");
+
+    setup(TestMode.TLS);
+
+    YBClient myClient = null;
+    // The mutliCA cert has two different root certs, with the first entry in the file
+    // being the pseudo root, which is not the one the server certs have been signed
+    // with.
+    String multiCA = String.format("%s/%s", certsDir(), "multiCA.crt");
+
+    AsyncYBClient aClient = new AsyncYBClient.AsyncYBClientBuilder(masterAddresses)
+                            .sslCertFile(multiCA)
+                            .build();
+    myClient = new YBClient(aClient);
+    myClient.waitForMasterLeader(Timeouts.adjustTimeoutSecForBuildType(10000));
+    myClient.close();
+    myClient = null;
+  }
+
   /**
    * Test to check that client connection fails when SSL is enabled but yb-client has SSL disabled.
    * @throws Exception