Permalink
Browse files

[#541] Play is no longer hardcoded to use utf-8 when communicating wi…

…th client.

Http.Request now uses encoding specified in HTTP-Request or fallback to Play.defaultWebEncoding which default is utf-8.
Http.Response can have different encoding for each response. It uses Play.defaultWebEncoding (default to utf-8) as default encoding for each response.
You can modify the encoding for a specific request like this: Http.Response.current().encoding = "iso-8859-1"

play.libs.WS is now fully encoding aware: default encoding for all WS-operations is now Play.defaultWebEncoding which default is set to utf-8.
If developer want to use WS with other than default encoding, this can be done like this:
WS.withEncoding("iso-8859-1").url(someUrl).get().getString()

Play.defaultWebEncoding can be configured in application.conf with the property 'application.web_encoding'
  • Loading branch information...
1 parent 0777f69 commit 61d6b44b8e0307b842af2dc50eead5b7dd49684d @mbknor mbknor committed Mar 13, 2011
Showing with 800 additions and 309 deletions.
  1. +3 −0 documentation/cheatsheets/controllers/ch08-ControllerLibraries.textile
  2. +19 −0 documentation/manual/controllers.textile
  3. BIN framework/lib/async-http-client-1.6.1.jar
  4. BIN framework/lib/async-http-client-1.6.3.jar
  5. +19 −0 framework/src/play/Play.java
  6. +13 −2 framework/src/play/data/parsing/ApacheMultipartParser.java
  7. +2 −1 framework/src/play/data/parsing/TextParser.java
  8. +33 −57 framework/src/play/data/parsing/UrlEncodedParser.java
  9. +17 −1 framework/src/play/jobs/Job.java
  10. +3 −1 framework/src/play/libs/MimeTypes.java
  11. +155 −19 framework/src/play/libs/WS.java
  12. +97 −50 framework/src/play/libs/ws/WSAsync.java
  13. +31 −15 framework/src/play/libs/ws/WSUrlFetch.java
  14. +3 −2 framework/src/play/mvc/ActionInvoker.java
  15. +28 −3 framework/src/play/mvc/Http.java
  16. +10 −7 framework/src/play/mvc/Router.java
  17. +3 −2 framework/src/play/mvc/Scope.java
  18. +1 −1 framework/src/play/mvc/results/Error.java
  19. +1 −1 framework/src/play/mvc/results/Forbidden.java
  20. +1 −1 framework/src/play/mvc/results/NotFound.java
  21. +3 −2 framework/src/play/mvc/results/RenderBinary.java
  22. +1 −1 framework/src/play/mvc/results/RenderHtml.java
  23. +3 −2 framework/src/play/mvc/results/RenderJson.java
  24. +1 −1 framework/src/play/mvc/results/RenderTemplate.java
  25. +3 −2 framework/src/play/mvc/results/RenderText.java
  26. +1 −1 framework/src/play/mvc/results/RenderXml.java
  27. +7 −0 framework/src/play/mvc/results/Result.java
  28. +29 −21 framework/src/play/server/PlayHandler.java
  29. +6 −5 framework/src/play/server/ServletWrapper.java
  30. +3 −1 framework/src/play/templates/FastTags.java
  31. +6 −0 framework/src/play/templates/GroovyTemplate.java
  32. +2 −1 framework/src/play/templates/JavaExtensions.java
  33. +1 −1 framework/src/play/test/FunctionalTest.java
  34. +34 −0 framework/src/play/utils/HTTP.java
  35. +1 −1 framework/templates/errors/404.html
  36. +1 −1 framework/templates/errors/500.html
  37. +1 −1 framework/templates/tags/i18n.tag
  38. +2 −2 framework/templates/tags/script.tag
  39. +1 −1 framework/templates/tags/stylesheet.tag
  40. +2 −2 framework/templates/tags/welcome.html
  41. +2 −2 modules/console/app/views/console/db.html
  42. +2 −2 modules/console/app/views/console/index.html
  43. +2 −2 modules/console/app/views/console/repl.html
  44. +1 −1 modules/crud/app/views/CRUD/layout.html
  45. +4 −4 modules/docviewer/app/views/PlayDocumentation/cheatSheet.html
  46. +2 −2 modules/docviewer/app/views/PlayDocumentation/page.html
  47. +1 −1 modules/secure/app/views/Secure/layout.html
  48. +3 −3 modules/testrunner/app/views/TestRunner/index.html
  49. +1 −1 modules/testrunner/app/views/TestRunner/results.html
  50. +1 −1 modules/testrunner/app/views/TestRunner/selenium-results.html
  51. +1 −1 resources/application-skel/app/views/errors/404.html
  52. +1 −1 resources/application-skel/app/views/errors/500.html
  53. +2 −2 resources/application-skel/app/views/main.html
  54. +1 −1 resources/application-skel/test/ApplicationTest.java
  55. +1 −1 samples-and-tests/booking/app/views/Application/index.html
  56. +2 −2 samples-and-tests/booking/app/views/Hotels/book.html
  57. +1 −1 samples-and-tests/booking/app/views/Hotels/index.html
  58. +1 −1 samples-and-tests/booking/app/views/errors/404.html
  59. +1 −1 samples-and-tests/booking/app/views/errors/500.html
  60. +3 −3 samples-and-tests/booking/app/views/main.html
  61. +1 −1 samples-and-tests/booking/test/ApplicationTest.java
  62. +1 −1 samples-and-tests/chat/app/views/Refresh/room.html
  63. +4 −4 samples-and-tests/chat/app/views/main.html
  64. +1 −1 samples-and-tests/facebook-oauth2/app/views/errors/404.html
  65. +1 −1 samples-and-tests/facebook-oauth2/app/views/errors/500.html
  66. +2 −2 samples-and-tests/facebook-oauth2/app/views/main.html
  67. +1 −1 samples-and-tests/facebook-oauth2/test/ApplicationTest.java
  68. +1 −1 samples-and-tests/forum/app/views/Application/login.html
  69. +1 −1 samples-and-tests/forum/app/views/Application/signup.html
  70. +1 −1 samples-and-tests/forum/app/views/Topics/post.html
  71. +1 −1 samples-and-tests/forum/app/views/Topics/reply.html
  72. +1 −1 samples-and-tests/forum/app/views/main.html
  73. +1 −1 samples-and-tests/jobboard/app/views/Application/main.html
  74. +1 −1 samples-and-tests/jobboard/app/views/errors/404.html
  75. +1 −1 samples-and-tests/jobboard/app/views/errors/500.html
  76. +3 −3 samples-and-tests/jobboard/app/views/main.html
  77. +26 −0 samples-and-tests/just-test-cases/app/controllers/CustomEncoding.java
  78. +46 −6 samples-and-tests/just-test-cases/app/controllers/Rest.java
  79. +2 −2 samples-and-tests/just-test-cases/app/views/Application/alertConfirmPrompt.html
  80. +1 −0 samples-and-tests/just-test-cases/app/views/CustomEncoding/getTemplate.txt
  81. +1 −1 samples-and-tests/just-test-cases/app/views/errors/404.html
  82. +1 −1 samples-and-tests/just-test-cases/app/views/errors/500.html
  83. +1 −1 samples-and-tests/just-test-cases/app/views/main.html
  84. +3 −0 samples-and-tests/just-test-cases/conf/application.conf
  85. +2 −0 samples-and-tests/just-test-cases/conf/routes
  86. +1 −0 samples-and-tests/just-test-cases/public/fileWithNoneStandardLetters_stored_in_iso_8859_1.html
  87. +1 −1 samples-and-tests/just-test-cases/test/ApplicationTest.java
  88. +18 −0 samples-and-tests/just-test-cases/test/CustomEncodingTest.java
  89. +41 −12 samples-and-tests/just-test-cases/test/RestTest.java
  90. +33 −0 samples-and-tests/just-test-cases/test/StaticContentTest.java
  91. +1 −1 samples-and-tests/twitter-oauth/app/views/errors/404.html
  92. +1 −1 samples-and-tests/twitter-oauth/app/views/errors/500.html
  93. +2 −2 samples-and-tests/twitter-oauth/app/views/main.html
  94. +1 −1 samples-and-tests/validation/app/views/main.html
  95. +1 −1 samples-and-tests/validation/test/application/SomeTests.java
  96. +1 −1 samples-and-tests/yabe/app/views/Application/show.html
  97. +1 −1 samples-and-tests/yabe/app/views/admin.html
  98. +1 −1 samples-and-tests/yabe/app/views/errors/404.html
  99. +1 −1 samples-and-tests/yabe/app/views/errors/500.html
  100. +1 −1 samples-and-tests/yabe/app/views/main.html
  101. +1 −1 samples-and-tests/yabe/test/ApplicationTest.java
  102. +1 −1 samples-and-tests/zencontact/app/views/Application/form.html
  103. +1 −1 samples-and-tests/zencontact/app/views/Application/index.html
  104. +1 −1 samples-and-tests/zencontact/app/views/Application/list.html
  105. +1 −1 samples-and-tests/zencontact/app/views/errors/404.html
  106. +1 −1 samples-and-tests/zencontact/app/views/errors/500.html
  107. +4 −4 samples-and-tests/zencontact/app/views/main.html
@@ -3,6 +3,9 @@ h2. Controller - Libraries
*==WS.url("http://s.com/posts").get().toJSON();==*
HTTP GET request to JSON
+*==WS.withEncoding("iso-8859-1").url("http://s.com/posts").get().toJSON();==*
+HTTP GET request to JSON using iso-8859-1 encoding
+
*==WS.url("http://s.com/").post().toXML();==*
HTTP POST request to XML
@@ -505,6 +505,25 @@ POST /clients Clients.create
* The browser then issues **GET /clients/3132**.
* …
+h3. Customizing web encoding
+
+Play! emphasize the use of UTF-8, but there might be situations where some responses must use different encoding,
+or that the whole app has to use other encoding.
+
+h4. Custom encoding for current Response
+
+To change encoding for current response, you can do it like this in your Controller:
+
+bc. response.encoding = "ISO-8859-1";
+
+h4. Custom web encoding for the entire App
+
+By setting the property **application.web_encoding** in application.conf, you can customize which encoding
+Play! uses when communicating with the browser. This also changes the default encoding for Plays powerful Web Service client WS.
+
+bc. application.web_encoding=ISO-8859-1
+
+When changing application.web_encoding, it affects the charset-part of the content-type-header. It also affects which encoding is used when transmitting rendered dynamic results, but it **does not** affect the bytes sent when Play! serves static content: So if you have modified the default response encoding and you have static text-files (in the public/ folder) that does contain special letters, you have to make sure that these files are stored according to the specified encoding. All other files should be stored in UTF-8.
h2. <a name="interceptions">Interceptions</a>
Binary file not shown.
Binary file not shown.
@@ -167,6 +167,11 @@ public boolean isProd() {
public static boolean lazyLoadTemplates = false;
/**
+ * This is used as default encoding everywhere related to the web: request, response, WS
+ */
+ public static String defaultWebEncoding = "utf-8";
+
+ /**
* Init the framework
*
* @param root The application path
@@ -448,6 +453,20 @@ public void run(){
Logger.warn("No secret key defined. Sessions will not be encrypted");
}
+ // Default web encoding
+ String _defaultWebEncoding = configuration.getProperty("application.web_encoding");
+ if( _defaultWebEncoding != null ) {
+ Logger.info("Using custom default web encoding: " + _defaultWebEncoding);
+ defaultWebEncoding = _defaultWebEncoding;
+ // Must update current response also, since the request/response triggering
+ // this configuration-loading in dev-mode have already been
+ // set up with the previous encoding
+ if( Http.Response.current() != null ) {
+ Http.Response.current().encoding = _defaultWebEncoding;
+ }
+ }
+
+
// Try to load all classes
Play.classloader.getAllClasses();
@@ -32,6 +32,7 @@
import play.data.Upload;
import play.exceptions.UnexpectedException;
import play.mvc.Http.Request;
+import play.utils.HTTP;
/**
* From Apache commons fileupload.
@@ -540,7 +541,7 @@ public String toString() {
public Map<String, String[]> parse(InputStream body) {
Map<String, String[]> result = new HashMap<String, String[]>();
try {
- FileItemIteratorImpl iter = new FileItemIteratorImpl(body, Request.current().headers.get("content-type").value(), "UTF-8");
+ FileItemIteratorImpl iter = new FileItemIteratorImpl(body, Request.current().headers.get("content-type").value(), Request.current().encoding);
while (iter.hasNext()) {
FileItemStream item = iter.next();
FileItem fileItem = new AutoFileItem(item);
@@ -552,7 +553,17 @@ public String toString() {
throw new IOFileUploadException("Processing of " + MULTIPART_FORM_DATA + " request failed. " + e.getMessage(), e);
}
if (fileItem.isFormField()) {
- putMapEntry(result, fileItem.getFieldName(), fileItem.getString("UTF-8"));
+ // must resolve encoding
+ String _encoding = Request.current().encoding; // this is our default
+ String _contentType = fileItem.getContentType();
+ if( _contentType != null ) {
+ HTTP.ContentTypeWithEncoding contentTypeEncoding = HTTP.parseContentType(_contentType);
+ if( contentTypeEncoding.encoding != null ) {
+ _encoding = contentTypeEncoding.encoding;
+ }
+ }
+
+ putMapEntry(result, fileItem.getFieldName(), fileItem.getString( _encoding ));
} else {
@SuppressWarnings("unchecked") List<Upload> uploads = (List<Upload>) Request.current().args.get("__UPLOADS");
if (uploads == null) {
@@ -5,6 +5,7 @@
import java.util.HashMap;
import java.util.Map;
import play.exceptions.UnexpectedException;
+import play.mvc.Http;
public class TextParser extends DataParser {
@@ -18,7 +19,7 @@
os.write(b);
}
byte[] data = os.toByteArray();
- params.put("body", new String[] {new String(data, "utf-8")});
+ params.put("body", new String[] {new String(data, Http.Request.current().encoding)});
return params;
} catch (Exception e) {
throw new UnexpectedException(e);
@@ -4,10 +4,12 @@
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import play.exceptions.UnexpectedException;
+import play.mvc.Http;
import play.utils.Utils;
/**
@@ -19,7 +21,8 @@
public static Map<String, String[]> parse(String urlEncoded) {
try {
- return new UrlEncodedParser().parse(new ByteArrayInputStream(urlEncoded.getBytes("utf-8")));
+ final String encoding = Http.Request.current().encoding;
+ return new UrlEncodedParser().parse(new ByteArrayInputStream(urlEncoded.getBytes( encoding )));
} catch (UnsupportedEncodingException ex) {
throw new UnexpectedException(ex);
}
@@ -33,77 +36,50 @@
@Override
public Map<String, String[]> parse(InputStream is) {
+ final String encoding = Http.Request.current().encoding;
try {
Map<String, String[]> params = new HashMap<String, String[]>();
ByteArrayOutputStream os = new ByteArrayOutputStream();
- int b;
- while ((b = is.read()) != -1) {
- os.write(b);
+ byte[] buffer = new byte[1024];
+ int bytesRead;
+ while ( (bytesRead = is.read(buffer)) > 0 ) {
+ os.write( buffer, 0, bytesRead);
}
- byte[] data = os.toByteArray();
+
+ String data = new String(os.toByteArray(), encoding);
// add the complete body as a parameters
if(!forQueryString) {
- params.put("body", new String[] {new String(data, "utf-8")});
+ params.put("body", new String[] {data});
}
-
- int ix = 0;
- int ox = 0;
- String key = null;
- String value = null;
- while (ix < data.length) {
- byte c = data[ix++];
- switch ((char) c) {
- case '&':
- value = new String(data, 0, ox, "utf-8");
- if (key != null) {
- Utils.Maps.mergeValueInMap(params, key, value);
- key = null;
- } else {
- Utils.Maps.mergeValueInMap(params, value, (String) null);
- }
- ox = 0;
- break;
- case '=':
- if (key == null) {
- key = new String(data, 0, ox, "utf-8");
- ox = 0;
+
+ // data is o the form:
+ // a=b&b=c%12...
+ String[] keyValues = data.split("&");
+ for (String keyValue : keyValues) {
+ // split this key-value on '='
+ String[] parts = keyValue.split("=");
+ // sanity check
+ if (parts.length >= 1) {
+ String key = URLDecoder.decode(parts[0],encoding);
+ if (key.length()>0) {
+ String value = null;
+ if (parts.length == 2) {
+ value = URLDecoder.decode(parts[1],encoding);
} else {
- data[ox++] = c;
+ // if keyValue ends with "=", then we have an empty value
+ // if not ending with "=", we have a key without a value (a flag)
+ if (keyValue.endsWith("=")) {
+ value = "";
+ }
}
- break;
- case '+':
- data[ox++] = (byte) ' ';
- break;
- case '%':
- data[ox++] = (byte) ((convertHexDigit(data[ix++]) << 4) + convertHexDigit(data[ix++]));
- break;
- default:
- data[ox++] = c;
+ Utils.Maps.mergeValueInMap(params, key, value);
+ }
}
}
- //The last value does not end in '&'. So save it now.
- value = new String(data, 0, ox, "utf-8");
- if (key != null) {
- Utils.Maps.mergeValueInMap(params, key, value);
- } else if (!value.isEmpty()) {
- Utils.Maps.mergeValueInMap(params, value, (String) null);
- }
return params;
} catch (Exception e) {
throw new UnexpectedException(e);
}
}
- private static byte convertHexDigit(byte b) {
- if ((b >= '0') && (b <= '9')) {
- return (byte) (b - '0');
- }
- if ((b >= 'a') && (b <= 'f')) {
- return (byte) (b - 'a' + 10);
- }
- if ((b >= 'A') && (b <= 'F')) {
- return (byte) (b - 'A' + 10);
- }
- return 0;
- }
}
@@ -150,9 +150,12 @@ public V call() {
monitor = MonitorFactory.start(getClass().getName()+".doJob()");
//Hack to enable template rendering with urls in jobs
- if( Http.Request.current.get() == null) {
+ if (Http.Request.current.get() == null) {
createFakeRequest();
}
+ if (Http.Response.current.get() == null) {
+ createFakeResponse();
+ }
result = doJobWithResult();
monitor.stop();
@@ -213,6 +216,19 @@ public Object loadObject() throws Exception {
}
+ /**
+ * If rendering with templates in a job, some template-operations require
+ * a current Response-object, eg: @@{...}}, because it needs to know which encoding to use..
+ * This method creates a fake one
+ */
+ private static void createFakeResponse() {
+
+ Http.Response fakeResponse = new Http.Response();
+ // now fakeResponse has default-encoding
+
+ Http.Response.current.set(fakeResponse);
+
+ }
@Override
@@ -1,6 +1,8 @@
package play.libs;
import play.*;
+import play.mvc.Http;
+
import java.io.InputStream;
import java.util.Enumeration;
import java.util.Properties;
@@ -73,7 +75,7 @@ public static String getContentType(String filename, String defaultContentType){
contentType = defaultContentType;
}
if (contentType != null && contentType.startsWith("text/")){
- return contentType + "; charset=utf-8";
+ return contentType + "; charset=" + Http.Response.current().encoding;
}
return contentType;
}
Oops, something went wrong. Retry.

0 comments on commit 61d6b44

Please sign in to comment.