Fixed issue in jsoup.connect when extracting character set from conte…

…nt-type header; now supports quoted

charset declaration.

CHANGES

@@ -1,6 +1,9 @@
 jsoup changelog
 
 *** Release 1.3.4 [2010-Sep-19]
+ * Fixed issue in jsoup.connect when extracting character set from content-type header; now supports quoted
+   charset declaration.
+
  * Fixed support for jsoup.connect to follow redirects between http & https URLs.
    <http://github.com/jhy/jsoup/issues/issue/37>
 

src/main/java/org/jsoup/helper/DataUtil.java

@@ -15,7 +15,7 @@
  *
  */
 public class DataUtil {
-    private static final Pattern charsetPattern = Pattern.compile("(?i)\\bcharset=([^\\s;]*)");
+    private static final Pattern charsetPattern = Pattern.compile("(?i)\\bcharset=\\s*\"?([^\\s;\"]*)");
     static final String defaultCharset = "UTF-8"; // used if not found in header or meta charset
     private static final int bufferSize = 0x20000; // ~130K.
 

src/main/java/org/jsoup/helper/HttpConnection.java

@@ -21,8 +21,6 @@
  * @see org.jsoup.Jsoup#connect(String) 
  */
 public class HttpConnection implements Connection {
-    private static final Pattern charsetPattern = Pattern.compile("(?i)\\bcharset=([^\\s;]*)");
-
     public static Connection connect(String url) {
         Connection con = new HttpConnection();
         con.url(url);
@@ -341,7 +339,7 @@ static Response execute(Connection.Request req) throws IOException {
                         new BufferedInputStream(new GZIPInputStream(conn.getInputStream())) :
                         new BufferedInputStream(conn.getInputStream());
                 res.byteData = DataUtil.readToByteBuffer(inStream);
-                res.charset = getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
+                res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
             } finally {
                 if (inStream != null) inStream.close();
             }
@@ -542,20 +540,4 @@ public String toString() {
             return key + "=" + value;
         }      
     }
-
-    /**
-     * Parse out a charset from a content type header.
-     *
-     * @param contentType e.g. "text/html; charset=EUC-JP"
-     * @return "EUC-JP", or null if not found. Charset is trimmed and uppercased.
-     */
-    private static String getCharsetFromContentType(String contentType) {
-        if (contentType == null) return null;
-
-        Matcher m = charsetPattern.matcher(contentType);
-        if (m.find()) {
-            return m.group(1).trim().toUpperCase();
-        }
-        return null;
-    }
 }

src/test/java/org/jsoup/helper/DataUtilTest.java

@@ -12,4 +12,10 @@ public void testCharset() {
         assertEquals(null, DataUtil.getCharsetFromContentType("text/html"));
         assertEquals(null, DataUtil.getCharsetFromContentType(null));
     }
+
+    @Test public void testQuotedCharset() {
+        assertEquals("UTF-8", DataUtil.getCharsetFromContentType("text/html; charset=\"utf-8\""));
+        assertEquals("UTF-8", DataUtil.getCharsetFromContentType("text/html;charset=\"utf-8\""));
+        assertEquals("ISO-8859-1", DataUtil.getCharsetFromContentType("text/html; charset=\"ISO-8859-1\""));
+    }
 }