From b1e960d4b0c1be1ce631c0ff20888fc0f4392598 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Tue, 2 Mar 2021 12:47:15 -0800 Subject: [PATCH 1/4] fix: default application/json charset to utf-8 --- .../google/api/client/http/HttpResponse.java | 18 +++++++++-- .../api/client/http/HttpResponseTest.java | 32 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java index efc3d1e58..889b7c389 100644 --- a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java +++ b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java @@ -18,6 +18,8 @@ import com.google.api.client.util.LoggingInputStream; import com.google.api.client.util.Preconditions; import com.google.api.client.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; + import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.IOException; @@ -512,8 +514,18 @@ public String parseAsString() throws IOException { * @since 1.10 */ public Charset getContentCharset() { - return mediaType == null || mediaType.getCharsetParameter() == null - ? StandardCharsets.ISO_8859_1 - : mediaType.getCharsetParameter(); + if (mediaType != null) { + // use specified charset parameter from content/type header if available + if (mediaType.getCharsetParameter() != null) { + return mediaType.getCharsetParameter(); + } + // fallback to well-known charsets + if (mediaType.getType().equals("application") && mediaType.getSubType().equals("json")) { + // https://tools.ietf.org/html/rfc4627 - JSON must be encoded with UTF-8 + return StandardCharsets.UTF_8; + } + } + return StandardCharsets.ISO_8859_1; } + } diff --git a/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java b/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java index 267d13caa..bfc09b6d2 100644 --- a/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java +++ b/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java @@ -58,10 +58,12 @@ public void testParseAsString_none() throws Exception { private static final String SAMPLE = "123\u05D9\u05e0\u05D9\u05D1"; private static final String SAMPLE2 = "123abc"; + private static final String JSON_SAMPLE = "{\"foo\": \"ßar\"}"; private static final String VALID_CONTENT_TYPE = "text/plain"; private static final String VALID_CONTENT_TYPE_WITH_PARAMS = "application/vnd.com.google.datastore.entity+json; charset=utf-8; version=v1; q=0.9"; private static final String INVALID_CONTENT_TYPE = "!!!invalid!!!"; + private static final String JSON_CONTENT_TYPE = "application/json"; public void testParseAsString_utf8() throws Exception { HttpTransport transport = @@ -83,6 +85,7 @@ public LowLevelHttpResponse execute() throws IOException { transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); HttpResponse response = request.execute(); assertEquals(SAMPLE, response.parseAsString()); + assertEquals("UTF-8", response.getContentCharset().name()); } public void testParseAsString_noContentType() throws Exception { @@ -104,6 +107,7 @@ public LowLevelHttpResponse execute() throws IOException { transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); HttpResponse response = request.execute(); assertEquals(SAMPLE2, response.parseAsString()); + assertEquals("ISO-8859-1", response.getContentCharset().name()); } public void testParseAsString_validContentType() throws Exception { @@ -129,6 +133,7 @@ public LowLevelHttpResponse execute() throws IOException { assertEquals(SAMPLE2, response.parseAsString()); assertEquals(VALID_CONTENT_TYPE, response.getContentType()); assertNotNull(response.getMediaType()); + assertEquals("ISO-8859-1", response.getContentCharset().name()); } public void testParseAsString_validContentTypeWithParams() throws Exception { @@ -154,6 +159,7 @@ public LowLevelHttpResponse execute() throws IOException { assertEquals(SAMPLE2, response.parseAsString()); assertEquals(VALID_CONTENT_TYPE_WITH_PARAMS, response.getContentType()); assertNotNull(response.getMediaType()); + assertEquals("UTF-8", response.getContentCharset().name()); } public void testParseAsString_invalidContentType() throws Exception { @@ -179,6 +185,32 @@ public LowLevelHttpResponse execute() throws IOException { assertEquals(SAMPLE2, response.parseAsString()); assertEquals(INVALID_CONTENT_TYPE, response.getContentType()); assertNull(response.getMediaType()); + assertEquals("ISO-8859-1", response.getContentCharset().name()); + } + + public void testParseAsString_jsonContentType() throws IOException { + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse result = new MockLowLevelHttpResponse(); + result.setContent(JSON_SAMPLE); + result.setContentType(JSON_CONTENT_TYPE); + return result; + } + }; + } + }; + HttpRequest request = + transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); + + HttpResponse response = request.execute(); + assertEquals(JSON_SAMPLE, response.parseAsString()); + assertEquals(JSON_CONTENT_TYPE, response.getContentType()); + assertEquals("UTF-8", response.getContentCharset().name()); } public void testStatusCode_negative_dontThrowException() throws Exception { From 0dd63c9f3a00e48dcb52f1698948c48bdebcab97 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Tue, 2 Mar 2021 12:58:53 -0800 Subject: [PATCH 2/4] fix: invert equality check to avoid NPE --- .../src/main/java/com/google/api/client/http/HttpResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java index 889b7c389..56c4ad1a7 100644 --- a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java +++ b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java @@ -520,7 +520,7 @@ public Charset getContentCharset() { return mediaType.getCharsetParameter(); } // fallback to well-known charsets - if (mediaType.getType().equals("application") && mediaType.getSubType().equals("json")) { + if ("application".equals(mediaType.getType()) && "json".equals(mediaType.getSubType())) { // https://tools.ietf.org/html/rfc4627 - JSON must be encoded with UTF-8 return StandardCharsets.UTF_8; } From b57091d3954eeea78039f299ce0e67e6c75e3e13 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Tue, 2 Mar 2021 12:59:51 -0800 Subject: [PATCH 3/4] revert: VisibleForTesting was not used --- .../src/main/java/com/google/api/client/http/HttpResponse.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java index 56c4ad1a7..33412fec6 100644 --- a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java +++ b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java @@ -18,7 +18,6 @@ import com.google.api.client.util.LoggingInputStream; import com.google.api.client.util.Preconditions; import com.google.api.client.util.StringUtils; -import com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayOutputStream; import java.io.EOFException; From 9505eae99119bbe800bc797ef11ed98c95d9573a Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Tue, 2 Mar 2021 13:00:20 -0800 Subject: [PATCH 4/4] chore: fix lint --- .../src/main/java/com/google/api/client/http/HttpResponse.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java index 33412fec6..37f4d7f11 100644 --- a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java +++ b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java @@ -18,7 +18,6 @@ import com.google.api.client.util.LoggingInputStream; import com.google.api.client.util.Preconditions; import com.google.api.client.util.StringUtils; - import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.IOException; @@ -526,5 +525,4 @@ public Charset getContentCharset() { } return StandardCharsets.ISO_8859_1; } - }