8369950: TLS connection to IPv6 address fails with BCJSSE due to IllegalArgumentException

Co-authored-by: Mikhail Yankelevich <myankelevich@openjdk.org>
Reviewed-by: djelinski, vyazici, dfuchs, myankelevich

Author: 2 people

src/java.base/share/classes/sun/net/www/protocol/https/HttpsClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -44,6 +44,7 @@
 import java.util.StringTokenizer;
 
 import javax.net.ssl.*;
+import sun.net.util.IPAddressUtil;
 import sun.net.www.http.HttpClient;
 import sun.net.www.protocol.http.AuthCacheImpl;
 import sun.net.www.protocol.http.HttpURLConnection;
@@ -471,7 +472,13 @@ public void afterConnect() throws IOException, UnknownHostException {
                     SSLParameters parameters = s.getSSLParameters();
                     parameters.setEndpointIdentificationAlgorithm("HTTPS");
                     // host has been set previously for SSLSocketImpl
-                    if (!(s instanceof SSLSocketImpl)) {
+                    if (!(s instanceof SSLSocketImpl) &&
+                            !IPAddressUtil.isIPv4LiteralAddress(host) &&
+                            !(host.charAt(0) == '[' && host.charAt(host.length() - 1) == ']' &&
+                                IPAddressUtil.isIPv6LiteralAddress(host.substring(1, host.length() - 1))
+                            )) {
+                        // Fully qualified DNS hostname of the server, as per section 3, RFC 6066
+                        // Literal IPv4 and IPv6 addresses are not permitted in "HostName".
                         parameters.setServerNames(List.of(new SNIHostName(host)));
                     }
                     s.setSSLParameters(parameters);