Introduce MIXED SSL Provider

handler/src/main/java/io/netty/handler/ssl/MixedOpenSslClientContext.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.internal.tcnative.SSL;
+
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.TrustManagerFactory;
+
+import static io.netty.handler.ssl.ReferenceCountedOpenSslClientContext.newSessionContext;
+
+/**
+ * A client-side {@link SslContext} which uses OpenSSL's SSL/TLS implementation.
+ *
+ * This class implements a MIXED mode for OpenSSL engines and contexts. It will use a finalizer to
+ * ensure native resources backing this context are automatically cleaned up, but it will rely on
+ * manual release for its produced {@link ReferenceCountedOpenSslEngine} instances.
+ */
+public final class MixedOpenSslClientContext extends MixedOpenSslContext {
+  private final OpenSslSessionContext sessionContext;
+
+  MixedOpenSslClientContext(X509Certificate[] trustCertCollection, TrustManagerFactory trustManagerFactory,
+                       X509Certificate[] keyCertChain, PrivateKey key, String keyPassword,
+                       KeyManagerFactory keyManagerFactory, Iterable<String> ciphers,
+                       CipherSuiteFilter cipherFilter, ApplicationProtocolConfig apn, String[] protocols,
+                       long sessionCacheSize, long sessionTimeout, boolean enableOcsp, String keyStore)
+      throws SSLException {
+    super(ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout, SSL.SSL_MODE_CLIENT, keyCertChain,
+        ClientAuth.NONE, protocols, false, enableOcsp);
+    boolean success = false;
+    try {
+      OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
+      sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
+          keyCertChain, key, keyPassword, keyManagerFactory, keyStore);
+      success = true;
+    } finally {
+      if (!success) {
+        release();
+      }
+    }
+  }
+
+  @Override
+  public OpenSslSessionContext sessionContext() {
+    return sessionContext;
+  }
+}

handler/src/main/java/io/netty/handler/ssl/MixedOpenSslContext.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.buffer.ByteBufAllocator;
+
+import java.security.cert.Certificate;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+
+/**
+ * This class implements a MIXED mode for OpenSSL engines and contexts. It will use a finalizer to
+ * ensure native resources backing this context are automatically cleaned up, but it will rely on
+ * manual release for its produced {@link ReferenceCountedOpenSslEngine} instances.
+ */
+public abstract class MixedOpenSslContext extends ReferenceCountedOpenSslContext {
+  MixedOpenSslContext(Iterable<String> ciphers, CipherSuiteFilter cipherFilter, ApplicationProtocolConfig apnCfg,
+                 long sessionCacheSize, long sessionTimeout, int mode, Certificate[] keyCertChain,
+                 ClientAuth clientAuth, String[] protocols, boolean startTls, boolean enableOcsp)
+      throws SSLException {
+    super(ciphers, cipherFilter, apnCfg, sessionCacheSize, sessionTimeout, mode, keyCertChain,
+        clientAuth, protocols, startTls, enableOcsp, false);
+  }
+
+  @Override
+  final SSLEngine newEngine0(ByteBufAllocator alloc, String peerHost, int peerPort, boolean jdkCompatibilityMode) {
+    return new ReferenceCountedOpenSslEngine(this, alloc, peerHost, peerPort, jdkCompatibilityMode, true);
+  }
+
+  @Override
+  @SuppressWarnings("FinalizeDeclaration")
+  protected final void finalize() throws Throwable {
+    super.finalize();
+    OpenSsl.releaseIfNeeded(this);
+  }
+}

handler/src/main/java/io/netty/handler/ssl/MixedOpenSslServerContext.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.internal.tcnative.SSL;
+
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.TrustManagerFactory;
+
+import static io.netty.handler.ssl.ReferenceCountedOpenSslServerContext.newSessionContext;
+
+/**
+ * A server-side {@link SslContext} which uses OpenSSL's SSL/TLS implementation.
+ *
+ * This class implements a MIXED mode for OpenSSL engines and contexts. It will use a finalizer to
+ * ensure native resources backing this context are automatically cleaned up, but it will rely on
+ * manual release for its produced {@link ReferenceCountedOpenSslEngine} instances.
+ */
+public final class MixedOpenSslServerContext extends OpenSslContext {
+  private final OpenSslServerSessionContext sessionContext;
+
+  MixedOpenSslServerContext(
+      X509Certificate[] trustCertCollection, TrustManagerFactory trustManagerFactory,
+      X509Certificate[] keyCertChain, PrivateKey key, String keyPassword, KeyManagerFactory keyManagerFactory,
+      Iterable<String> ciphers, CipherSuiteFilter cipherFilter, ApplicationProtocolConfig apn,
+      long sessionCacheSize, long sessionTimeout, ClientAuth clientAuth, String[] protocols, boolean startTls,
+      boolean enableOcsp, String keyStore) throws SSLException {
+    super(ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout, SSL.SSL_MODE_SERVER, keyCertChain,
+        clientAuth, protocols, startTls, enableOcsp);
+
+    boolean success = false;
+    try {
+      OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
+      sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
+          keyCertChain, key, keyPassword, keyManagerFactory, keyStore);
+      success = true;
+    } finally {
+      if (!success) {
+        release();
+      }
+    }
+  }
+
+  @Override
+  public OpenSslServerSessionContext sessionContext() {
+    return sessionContext;
+  }
+}

handler/src/main/java/io/netty/handler/ssl/SslContext.java

@@ -475,6 +475,14 @@ static SslContext newServerContextInternal(
                     trustCertCollection, trustManagerFactory, keyCertChain, key, keyPassword,
                     keyManagerFactory, ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout,
                     clientAuth, protocols, startTls, enableOcsp, keyStoreType);
+
+        case OPENSSL_MIXED:
+            verifyNullSslContextProvider(provider, sslContextProvider);
+            return new MixedOpenSslServerContext(
+                trustCertCollection, trustManagerFactory, keyCertChain, key, keyPassword,
+                keyManagerFactory, ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout,
+                clientAuth, protocols, startTls, enableOcsp, keyStoreType);
+
         default:
             throw new Error(provider.toString());
         }
@@ -831,6 +839,14 @@ static SslContext newClientContextInternal(
                         trustCert, trustManagerFactory, keyCertChain, key, keyPassword,
                         keyManagerFactory, ciphers, cipherFilter, apn, protocols, sessionCacheSize, sessionTimeout,
                         enableOcsp, keyStoreType);
+
+            case OPENSSL_MIXED:
+                verifyNullSslContextProvider(provider, sslContextProvider);
+                return new MixedOpenSslClientContext(
+                    trustCert, trustManagerFactory, keyCertChain, key, keyPassword,
+                    keyManagerFactory, ciphers, cipherFilter, apn, protocols, sessionCacheSize, sessionTimeout,
+                    enableOcsp, keyStoreType);
+
             default:
                 throw new Error(provider.toString());
         }

handler/src/main/java/io/netty/handler/ssl/SslProvider.java

@@ -35,5 +35,10 @@ public enum SslProvider {
      * OpenSSL-based implementation which does not have finalizers and instead implements {@link ReferenceCounted}.
      */
     @UnstableApi
-    OPENSSL_REFCNT
+    OPENSSL_REFCNT,
+    /**
+     * OpenSSL-based implementation which mixes {@link ReferenceCounted} engines with finalizer-based contexts.
+     */
+    @UnstableApi
+    OPENSSL_MIXED
 }