[#541] Play is no longer hardcoded to use utf-8 when communicating wi…

…th client.

Http.Request now uses encoding specified in HTTP-Request or fallback to Play.defaultWebEncoding which default is utf-8.
Http.Response can have different encoding for each response. It uses Play.defaultWebEncoding (default to utf-8) as default encoding for each response.
You can modify the encoding for a specific request like this: Http.Response.current().encoding = "iso-8859-1"

play.libs.WS is now fully encoding aware: default encoding for all WS-operations is now Play.defaultWebEncoding which default is set to utf-8.
If developer want to use WS with other than default encoding, this can be done like this:
WS.withEncoding("iso-8859-1").url(someUrl).get().getString()

Play.defaultWebEncoding can be configured in application.conf with the property 'application.web_encoding'

documentation/cheatsheets/controllers/ch08-ControllerLibraries.textile

@@ -3,6 +3,9 @@ h2. Controller - Libraries
 *==WS.url("http://s.com/posts").get().toJSON();==*
 HTTP GET request to JSON
 
+*==WS.withEncoding("iso-8859-1").url("http://s.com/posts").get().toJSON();==*
+HTTP GET request to JSON using iso-8859-1 encoding
+
 *==WS.url("http://s.com/").post().toXML();==*
 HTTP POST request to XML
 

documentation/manual/controllers.textile

@@ -505,6 +505,25 @@ POST   /clients                 Clients.create
 * The browser then issues **GET /clients/3132**.
 * …
 
+h3. Customizing web encoding
+
+Play! emphasize the use of UTF-8, but there might be situations where some responses must use different encoding,
+or that the whole app has to use other encoding.
+
+h4. Custom encoding for current Response
+
+To change encoding for current response, you can do it like this in your Controller:
+
+bc. response.encoding = "ISO-8859-1";
+
+h4. Custom web encoding for the entire App
+
+By setting the property **application.web_encoding** in application.conf, you can customize which encoding
+Play! uses when communicating with the browser. This also changes the default encoding for Plays powerful Web Service client WS.
+
+bc. application.web_encoding=ISO-8859-1
+
+When changing application.web_encoding, it affects the charset-part of the content-type-header. It also affects which encoding is used when transmitting rendered dynamic results, but it **does not** affect the bytes sent when Play! serves static content: So if you have modified the default response encoding and you have static text-files (in the public/ folder) that does contain special letters, you have to make sure that these files are stored according to the specified encoding. All other files should be stored in UTF-8.
 
 h2. <a name="interceptions">Interceptions</a>
 

framework/src/play/Play.java

@@ -166,6 +166,11 @@ public boolean isProd() {
      */
     public static boolean lazyLoadTemplates = false;
 
+    /**
+     * This is used as default encoding everywhere related to the web: request, response, WS
+     */
+    public static String defaultWebEncoding = "utf-8";
+
     /**
      * Init the framework
      *
@@ -448,6 +453,20 @@ public void run(){
                 Logger.warn("No secret key defined. Sessions will not be encrypted");
             }
 
+            // Default web encoding
+            String _defaultWebEncoding = configuration.getProperty("application.web_encoding");
+            if( _defaultWebEncoding != null ) {
+                Logger.info("Using custom default web encoding: " + _defaultWebEncoding);
+                defaultWebEncoding = _defaultWebEncoding;
+                // Must update current response also, since the request/response triggering
+                // this configuration-loading in dev-mode have already been
+                // set up with the previous encoding
+                if( Http.Response.current() != null ) {
+                    Http.Response.current().encoding = _defaultWebEncoding;
+                }
+            }
+
+
             // Try to load all classes
             Play.classloader.getAllClasses();
 

framework/src/play/data/parsing/ApacheMultipartParser.java

@@ -32,6 +32,7 @@
 import play.data.Upload;
 import play.exceptions.UnexpectedException;
 import play.mvc.Http.Request;
+import play.utils.HTTP;
 
 /**
  * From Apache commons fileupload.
@@ -540,7 +541,7 @@ public String toString() {
     public Map<String, String[]> parse(InputStream body) {
         Map<String, String[]> result = new HashMap<String, String[]>();
         try {
-            FileItemIteratorImpl iter = new FileItemIteratorImpl(body, Request.current().headers.get("content-type").value(), "UTF-8");
+            FileItemIteratorImpl iter = new FileItemIteratorImpl(body, Request.current().headers.get("content-type").value(), Request.current().encoding);
             while (iter.hasNext()) {
                 FileItemStream item = iter.next();
                 FileItem fileItem = new AutoFileItem(item);
@@ -552,7 +553,17 @@ public Map<String, String[]> parse(InputStream body) {
                     throw new IOFileUploadException("Processing of " + MULTIPART_FORM_DATA + " request failed. " + e.getMessage(), e);
                 }
                 if (fileItem.isFormField()) {
-                    putMapEntry(result, fileItem.getFieldName(), fileItem.getString("UTF-8"));
+                    // must resolve encoding
+                    String _encoding = Request.current().encoding; // this is our default
+                    String _contentType = fileItem.getContentType();
+                    if( _contentType != null ) {
+                        HTTP.ContentTypeWithEncoding contentTypeEncoding = HTTP.parseContentType(_contentType);
+                        if( contentTypeEncoding.encoding != null ) {
+                            _encoding = contentTypeEncoding.encoding;
+                        }
+                    }
+
+                    putMapEntry(result, fileItem.getFieldName(), fileItem.getString( _encoding ));
                 } else {
                     @SuppressWarnings("unchecked") List<Upload> uploads = (List<Upload>) Request.current().args.get("__UPLOADS");
                     if (uploads == null) {

framework/src/play/data/parsing/TextParser.java

@@ -5,6 +5,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import play.exceptions.UnexpectedException;
+import play.mvc.Http;
 
 public class TextParser extends DataParser {
 
@@ -18,7 +19,7 @@ public Map<String, String[]> parse(InputStream is) {
                 os.write(b);
             }
             byte[] data = os.toByteArray();
-            params.put("body", new String[] {new String(data, "utf-8")});
+            params.put("body", new String[] {new String(data, Http.Request.current().encoding)});
             return params;
         } catch (Exception e) {
             throw new UnexpectedException(e);

framework/src/play/data/parsing/UrlEncodedParser.java

@@ -4,10 +4,12 @@
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
 import java.util.HashMap;
 import java.util.Map;
 
 import play.exceptions.UnexpectedException;
+import play.mvc.Http;
 import play.utils.Utils;
 
 /**
@@ -19,7 +21,8 @@ public class UrlEncodedParser extends DataParser {
 
     public static Map<String, String[]> parse(String urlEncoded) {
         try {
-            return new UrlEncodedParser().parse(new ByteArrayInputStream(urlEncoded.getBytes("utf-8")));
+            final String encoding = Http.Request.current().encoding;
+            return new UrlEncodedParser().parse(new ByteArrayInputStream(urlEncoded.getBytes( encoding )));
         } catch (UnsupportedEncodingException ex) {
             throw new UnexpectedException(ex);
         }
@@ -33,77 +36,50 @@ public static Map<String, String[]> parseQueryString(InputStream is) {
 
     @Override
     public Map<String, String[]> parse(InputStream is) {
+        final String encoding = Http.Request.current().encoding;
         try {
             Map<String, String[]> params = new HashMap<String, String[]>();
             ByteArrayOutputStream os = new ByteArrayOutputStream();
-            int b;
-            while ((b = is.read()) != -1) {
-                os.write(b);
+            byte[] buffer = new byte[1024];
+            int bytesRead;
+            while ( (bytesRead = is.read(buffer)) > 0 ) {
+                os.write( buffer, 0, bytesRead);
             }
-            byte[] data = os.toByteArray();
+
+            String data = new String(os.toByteArray(), encoding);
             // add the complete body as a parameters
             if(!forQueryString) {
-                params.put("body", new String[] {new String(data, "utf-8")});
+                params.put("body", new String[] {data});
             }
-
-            int ix = 0;
-            int ox = 0;
-            String key = null;
-            String value = null;
-            while (ix < data.length) {
-                byte c = data[ix++];
-                switch ((char) c) {
-                    case '&':
-                        value = new String(data, 0, ox, "utf-8");
-                        if (key != null) {
-                            Utils.Maps.mergeValueInMap(params, key, value);
-                            key = null;
-                        } else {
-                            Utils.Maps.mergeValueInMap(params, value, (String) null);
-                        }
-                        ox = 0;
-                        break;
-                    case '=':
-                        if (key == null) {
-                            key = new String(data, 0, ox, "utf-8");
-                            ox = 0;
+
+            // data is o the form:
+            // a=b&b=c%12...
+            String[] keyValues = data.split("&");
+            for (String keyValue : keyValues) {
+                // split this key-value on '='
+                String[] parts = keyValue.split("=");
+                // sanity check
+                if (parts.length >= 1) {
+                    String key = URLDecoder.decode(parts[0],encoding);
+                    if (key.length()>0) {
+                        String value = null;
+                        if (parts.length == 2) {
+                            value = URLDecoder.decode(parts[1],encoding);
                         } else {
-                            data[ox++] = c;
+                            // if keyValue ends with "=", then we have an empty value
+                            // if not ending with "=", we have a key without a value (a flag)
+                            if (keyValue.endsWith("=")) {
+                                value = "";
+                            }
                         }
-                        break;
-                    case '+':
-                        data[ox++] = (byte) ' ';
-                        break;
-                    case '%':
-                        data[ox++] = (byte) ((convertHexDigit(data[ix++]) << 4) + convertHexDigit(data[ix++]));
-                        break;
-                    default:
-                        data[ox++] = c;
+                        Utils.Maps.mergeValueInMap(params, key, value);
+                    }
                 }
             }
-            //The last value does not end in '&'.  So save it now.
-            value = new String(data, 0, ox, "utf-8");
-            if (key != null) {
-                Utils.Maps.mergeValueInMap(params, key, value);
-            } else if (!value.isEmpty()) {
-                Utils.Maps.mergeValueInMap(params, value, (String) null);
-            }
             return params;
         } catch (Exception e) {
             throw new UnexpectedException(e);
         }
     }
 
-    private static byte convertHexDigit(byte b) {
-        if ((b >= '0') && (b <= '9')) {
-            return (byte) (b - '0');
-        }
-        if ((b >= 'a') && (b <= 'f')) {
-            return (byte) (b - 'a' + 10);
-        }
-        if ((b >= 'A') && (b <= 'F')) {
-            return (byte) (b - 'A' + 10);
-        }
-        return 0;
-    }
 }

framework/src/play/jobs/Job.java

@@ -150,9 +150,12 @@ public V call() {
                     monitor = MonitorFactory.start(getClass().getName()+".doJob()");
 
                     //Hack to enable template rendering with urls in jobs
-                    if( Http.Request.current.get() == null) {
+                    if (Http.Request.current.get() == null) {
                         createFakeRequest();
                     }
+                    if (Http.Response.current.get() == null) {
+                        createFakeResponse();
+                    }
 
                     result = doJobWithResult();
                     monitor.stop();
@@ -213,6 +216,19 @@ public Object loadObject() throws Exception {
 
     }
 
+    /**
+     * If rendering with templates in a job, some template-operations require
+     * a current Response-object, eg: @@{...}}, because it needs to know which encoding to use..
+     * This method creates a fake one
+     */
+    private static void createFakeResponse() {
+
+        Http.Response fakeResponse = new Http.Response();
+        // now fakeResponse has default-encoding
+
+        Http.Response.current.set(fakeResponse);
+
+    }
 
 
     @Override

framework/src/play/libs/MimeTypes.java

@@ -1,6 +1,8 @@
 package play.libs;
 
 import play.*;
+import play.mvc.Http;
+
 import java.io.InputStream;
 import java.util.Enumeration;
 import java.util.Properties;
@@ -73,7 +75,7 @@ public static String getContentType(String filename, String defaultContentType){
     		contentType =  defaultContentType;
     	}
     	if (contentType != null && contentType.startsWith("text/")){
-    		return contentType + "; charset=utf-8";
+    		return contentType + "; charset=" + Http.Response.current().encoding;
     	}
     	return contentType;
     }