8254631: Better support ALPN byte wire values in SunJSSE

Reviewed-by: dcherepanov
Backport-of: fe5cccc1ec76a5c29b1f55af311823f84483395b

src/java.base/share/classes/sun/security/ssl/AlpnExtension.java

@@ -27,7 +27,10 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
+import java.nio.charset.Charset;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.Security;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -59,6 +62,20 @@ final class AlpnExtension {
 
     static final SSLStringizer alpnStringizer = new AlpnStringizer();
 
+    // Encoding Charset to convert between String and byte[]
+    static final Charset alpnCharset;
+
+    static {
+        String alpnCharsetString = AccessController.doPrivileged(
+                (PrivilegedAction<String>) ()
+                        -> Security.getProperty("jdk.tls.alpnCharset"));
+        if ((alpnCharsetString == null)
+                || (alpnCharsetString.length() == 0)) {
+            alpnCharsetString = "ISO_8859_1";
+        }
+        alpnCharset = Charset.forName(alpnCharsetString);
+    }
+
     /**
      * The "application_layer_protocol_negotiation" extension.
      *
@@ -101,7 +118,7 @@ private AlpnSpec(HandshakeContext hc,
                         "extension: empty application protocol name"));
                 }
 
-                String appProtocol = new String(bytes, StandardCharsets.UTF_8);
+                String appProtocol = new String(bytes, alpnCharset);
                 protocolNames.add(appProtocol);
             }
 
@@ -168,10 +185,10 @@ public byte[] produce(ConnectionContext context,
                 return null;
             }
 
-            // Produce the extension.
+            // Produce the extension:  first find the overall length
             int listLength = 0;     // ProtocolNameList length
             for (String ap : laps) {
-                int length = ap.getBytes(StandardCharsets.UTF_8).length;
+                int length = ap.getBytes(alpnCharset).length;
                 if (length == 0) {
                     // log the configuration problem
                     if (SSLLogger.isOn && SSLLogger.isOn("ssl,handshake")) {
@@ -223,8 +240,10 @@ public byte[] produce(ConnectionContext context,
             byte[] extData = new byte[listLength + 2];
             ByteBuffer m = ByteBuffer.wrap(extData);
             Record.putInt16(m, listLength);
+
+            // opaque ProtocolName<1..2^8-1>;
             for (String ap : laps) {
-                Record.putBytes8(m, ap.getBytes(StandardCharsets.UTF_8));
+                Record.putBytes8(m, ap.getBytes(alpnCharset));
             }
 
             // Update the context.
@@ -414,14 +433,14 @@ public byte[] produce(ConnectionContext context,
             }
 
             // opaque ProtocolName<1..2^8-1>, RFC 7301.
-            int listLen = shc.applicationProtocol.length() + 1;
-                                                        // 1: length byte
+            byte[] bytes = shc.applicationProtocol.getBytes(alpnCharset);
+            int listLen = bytes.length + 1;             // 1: length byte
+
             // ProtocolName protocol_name_list<2..2^16-1>, RFC 7301.
             byte[] extData = new byte[listLen + 2];     // 2: list length
             ByteBuffer m = ByteBuffer.wrap(extData);
             Record.putInt16(m, listLen);
-            Record.putBytes8(m,
-                    shc.applicationProtocol.getBytes(StandardCharsets.UTF_8));
+            Record.putBytes8(m, bytes);
 
             // Update the context.
             shc.conContext.applicationProtocol = shc.applicationProtocol;

src/java.base/share/conf/security/java.security

@@ -1321,6 +1321,16 @@ jdk.io.permissionsUseCanonicalPath=false
 #
 #jdk.security.allowNonCaAnchor=true
 
+#
+# The default Character set name (java.nio.charset.Charset.forName())
+# for converting TLS ALPN values between byte arrays and Strings.
+# Prior versions of the JDK may use UTF-8 as the default charset. If
+# you experience interoperability issues, setting this property to UTF-8
+# may help.
+#
+# jdk.tls.alpnCharset=UTF-8
+jdk.tls.alpnCharset=ISO_8859_1
+
 #
 # JNDI Object Factories Filter
 #