Merge branch 'pr/343'

src/main/java/org/jsoup/Connection.java

@@ -120,6 +120,21 @@ public final boolean hasBody() {
      */
     public Connection ignoreContentType(boolean ignoreContentType);
 
+    /**
+     * Disable/enable TSL certificates validation for HTTPS requests.
+     * <p/>
+     * By default this is <b>true</b>; all
+     * connections over HTTPS perform normal validation of certificates, and will abort requests if the provided
+     * certificate does not validate.
+     * <p/>
+     * Some servers use expired, self-generated certificates; or your JDK may not
+     * support SNI hosts. In which case, you may want to enable this setting.
+     * <p/> <b>Be careful</b> and understand why you need to disable these validations.
+     * @param value if should validate TSL (SSL) certificates. <b>true</b> by default.
+     * @return this Connection, for chaining
+     */
+    Connection validateTLSCertificates(boolean value);
+
     /**
      * Add a request data parameter. Request parameters are sent in the request query string for GETs, and in the
      * request body for POSTs. A request may have multiple values of the same name.
@@ -376,6 +391,8 @@ interface Base<T extends Base> {
      * Represents a HTTP request.
      */
     public interface Request extends Base<Request> {
+
+
         /**
          * Get the request timeout, in milliseconds.
          * @return the timeout in milliseconds.
@@ -443,6 +460,18 @@ public interface Request extends Base<Request> {
          */
         public Request ignoreContentType(boolean ignoreContentType);
 
+        /**
+         * Get the current state of TLS (SSL) certificate validation.
+         * @return true if TLS cert validation enabled
+         */
+        boolean validateTLSCertificates();
+
+        /**
+         * Set TLS certificate validation.
+         * @param value set false to ignore TLS (SSL) certificates
+         */
+        void validateTLSCertificates(boolean value);
+
         /**
          * Add a data parameter to the request
          * @param keyval data to add.

src/main/java/org/jsoup/helper/HttpConnection.java

@@ -7,19 +7,21 @@
 import org.jsoup.parser.Parser;
 import org.jsoup.parser.TokenQueue;
 
+import javax.net.ssl.*;
 import java.io.*;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 
-import static org.jsoup.Connection.Method;
-
 /**
  * Implementation of {@link Connection}.
  * @see org.jsoup.Jsoup#connect(String)
@@ -119,6 +121,11 @@ public Connection ignoreContentType(boolean ignoreContentType) {
         return this;
     }
 
+    public Connection validateTLSCertificates(boolean value) {
+        req.validateTLSCertificates(value);
+        return this;
+    }
+
     public Connection data(String key, String value) {
         req.data(KeyVal.create(key, value));
         return this;
@@ -344,8 +351,9 @@ public static class Request extends HttpConnection.Base<Connection.Request> impl
         private boolean ignoreHttpErrors = false;
         private boolean ignoreContentType = false;
         private Parser parser;
+        private boolean validateTSLCertificates = true;
 
-      	private Request() {
+        private Request() {
             timeoutMilliseconds = 3000;
             maxBodySizeBytes = 1024 * 1024; // 1MB
             followRedirects = true;
@@ -388,6 +396,14 @@ public boolean ignoreHttpErrors() {
             return ignoreHttpErrors;
         }
 
+        public boolean validateTLSCertificates() {
+            return validateTSLCertificates;
+        }
+
+        public void validateTLSCertificates(boolean value) {
+            validateTSLCertificates = value;
+        }
+
         public Connection.Request ignoreHttpErrors(boolean ignoreHttpErrors) {
             this.ignoreHttpErrors = ignoreHttpErrors;
             return this;
@@ -424,6 +440,7 @@ public Parser parser() {
 
     public static class Response extends HttpConnection.Base<Connection.Response> implements Connection.Response {
         private static final int MAX_REDIRECTS = 20;
+        private static SSLSocketFactory sslSocketFactory;
         private static final String LOCATION = "Location";
         private int statusCode;
         private String statusMessage;
@@ -580,10 +597,20 @@ public byte[] bodyAsBytes() {
         // set up connection defaults, and details from request
         private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
             HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
+
             conn.setRequestMethod(req.method().name());
             conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
             conn.setConnectTimeout(req.timeout());
             conn.setReadTimeout(req.timeout());
+
+            if (conn instanceof HttpsURLConnection) {
+                if (!req.validateTLSCertificates()) {
+                    initUnSecureTSL();
+                    ((HttpsURLConnection)conn).setSSLSocketFactory(sslSocketFactory);
+                    ((HttpsURLConnection)conn).setHostnameVerifier(getInsecureVerifier());
+                }
+            }
+
             if (req.method().hasBody())
                 conn.setDoOutput(true);
             if (req.cookies().size() > 0)
@@ -594,6 +621,62 @@ private static HttpURLConnection createConnection(Connection.Request req) throws
             return conn;
         }
 
+        /**
+         * Instantiate Hostname Verifier that does nothing.
+         * This is used for connections with disabled SSL certificates validation.
+         *
+         *
+         * @return Hostname Verifier that does nothing and accepts all hostnames
+         */
+        private static HostnameVerifier getInsecureVerifier() {
+            return new HostnameVerifier() {
+                public boolean verify(String urlHostName, SSLSession session) {
+                    return true;
+                }
+            };
+        }
+
+        /**
+         * Initialise Trust manager that does not validate certificate chains and
+         * add it to current SSLContext.
+         * <p/>
+         * please not that this method will only perform action if sslSocketFactory is not yet
+         * instantiated.
+         *
+         * @throws IOException
+         */
+        private static synchronized void initUnSecureTSL() throws IOException {
+            if (sslSocketFactory == null) {
+                // Create a trust manager that does not validate certificate chains
+                final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
+
+                    public void checkClientTrusted(final X509Certificate[] chain, final String authType) {
+                    }
+
+                    public void checkServerTrusted(final X509Certificate[] chain, final String authType) {
+                    }
+
+                    public X509Certificate[] getAcceptedIssuers() {
+                        return null;
+                    }
+                }};
+
+                // Install the all-trusting trust manager
+                final SSLContext sslContext;
+                try {
+                    sslContext = SSLContext.getInstance("SSL");
+                    sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
+                    // Create an ssl socket factory with our all-trusting manager
+                    sslSocketFactory = sslContext.getSocketFactory();
+                } catch (NoSuchAlgorithmException e) {
+                    throw new IOException("Can't create unsecure trust manager");
+                } catch (KeyManagementException e) {
+                    throw new IOException("Can't create unsecure trust manager");
+                }
+            }
+
+        }
+
         // set up url, method, header, cookies
         private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException {
             method = Method.valueOf(conn.getRequestMethod());
@@ -630,7 +713,7 @@ void processResponseHeaders(Map<String, List<String>> resHeaders) {
                         String cookieVal = cd.consumeTo(";").trim();
                         if (cookieVal == null)
                             cookieVal = "";
-                        // ignores path, date, domain, secure et al. req'd?
+                        // ignores path, date, domain, validateTLSCertificates et al. req'd?
                         // name not blank, value not null
                         if (cookieName != null && cookieName.length() > 0)
                             cookie(cookieName, cookieVal);

src/test/java/org/jsoup/integration/UrlConnectTest.java

@@ -12,21 +12,22 @@
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Map;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.*;
 
 /**
  Tests the URL connection. Not enabled by default, so tests don't require network connection.
 
  @author Jonathan Hedley, jonathan@hedley.net */
 @Ignore // ignored by default so tests don't require network access. comment out to enable.
 public class UrlConnectTest {
+    private static final String WEBSITE_WITH_INVALID_CERTIFICATE = "https://certs.cac.washington.edu/CAtest/";
+    private static final String WEBSITE_WITH_SNI = "https://jsoup.org/";
     private static String echoURL = "http://direct.infohound.net/tools/q.pl";
 
     @Test
@@ -298,6 +299,63 @@ public void maxBodySize() throws IOException {
         assertEquals(actualDocText, unlimitedRes.parse().text().length());
     }
 
+    /**
+     * Verify that security disabling feature works properly.
+     * <p/>
+     * 1. try to hit url with invalid certificate and evaluate that exception is thrown
+     *
+     * @throws Exception
+     */
+    @Test(expected = IOException.class)
+    public void testUnsafeFail() throws Exception {
+        String url = WEBSITE_WITH_INVALID_CERTIFICATE;
+        Jsoup.connect(url).execute();
+    }
+
+
+    /**
+     * Verify that requests to websites with SNI fail on jdk 1.6
+     * <p/>
+     * read for more details:
+     * http://en.wikipedia.org/wiki/Server_Name_Indication
+     *
+     * Test is ignored independent from others as it requires JDK 1.6
+     * @throws Exception
+     */
+    @Test(expected = IOException.class)
+    public void testSNIFail() throws Exception {
+        String url = WEBSITE_WITH_SNI;
+        Jsoup.connect(url).execute();
+    }
+
+    /**
+     * Verify that requests to websites with SNI pass
+     * <p/>
+     * <b>NB!</b> this test is FAILING right now on jdk 1.6
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testSNIPass() throws Exception {
+        String url = WEBSITE_WITH_SNI;
+        Connection.Response defaultRes = Jsoup.connect(url).validateTLSCertificates(false).execute();
+        assertThat(defaultRes.statusCode(), is(200));
+    }
+
+    /**
+     * Verify that security disabling feature works properly.
+     * <p/>
+     * 1. disable security checks and call the same url to verify that content is consumed correctly
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testUnsafePass() throws Exception {
+        String url = WEBSITE_WITH_INVALID_CERTIFICATE;
+        Connection.Response defaultRes = Jsoup.connect(url).validateTLSCertificates(false).execute();
+        assertThat(defaultRes.statusCode(), is(200));
+    }
+
     @Test
     public void shouldWorkForCharsetInExtraAttribute() throws IOException {
         Connection.Response res = Jsoup.connect("https://www.creditmutuel.com/groupe/fr/").execute();