From 2c6e9ab69fa42f4b8f00c630a4c028d35671136c Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 12 Jul 2023 10:26:20 +0200 Subject: [PATCH] Core: Fix unicode handling in HTTPClient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #7821 Without the fix, tests would fail with ``` expected: struct<1: id: required int (unique ID 🤪), 2: data: required string> but was: struct<1: id: required int (unique ID ?), 2: data: required string> ``` --- core/src/main/java/org/apache/iceberg/rest/HTTPClient.java | 7 ++++--- .../test/java/org/apache/iceberg/catalog/CatalogTests.java | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java index 9af4b56797db..b2b6fc8a7c59 100644 --- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java @@ -24,6 +24,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -105,7 +106,7 @@ private static String extractResponseBodyAsString(CloseableHttpResponse response } // EntityUtils.toString returns null when HttpEntity.getContent returns null. - return EntityUtils.toString(response.getEntity(), "UTF-8"); + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); } catch (IOException | ParseException e) { throw new RESTException(e, "Failed to convert HTTP response body to string"); } @@ -471,13 +472,13 @@ public HTTPClient build() { private StringEntity toJson(Object requestBody) { try { - return new StringEntity(mapper.writeValueAsString(requestBody)); + return new StringEntity(mapper.writeValueAsString(requestBody), StandardCharsets.UTF_8); } catch (JsonProcessingException e) { throw new RESTException(e, "Failed to write request body: %s", requestBody); } } private StringEntity toFormEncoding(Map formData) { - return new StringEntity(RESTUtil.encodeFormData(formData)); + return new StringEntity(RESTUtil.encodeFormData(formData), StandardCharsets.UTF_8); } } diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java index 15259d979e18..1b274328564e 100644 --- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java @@ -73,19 +73,19 @@ public abstract class CatalogTests { // Schema passed to create tables protected static final Schema SCHEMA = new Schema( - required(3, "id", Types.IntegerType.get(), "unique ID"), + required(3, "id", Types.IntegerType.get(), "unique ID 🤪"), required(4, "data", Types.StringType.get())); // This is the actual schema for the table, with column IDs reassigned private static final Schema TABLE_SCHEMA = new Schema( - required(1, "id", Types.IntegerType.get(), "unique ID"), + required(1, "id", Types.IntegerType.get(), "unique ID 🤪"), required(2, "data", Types.StringType.get())); // This is the actual schema for the table, with column IDs reassigned private static final Schema REPLACE_SCHEMA = new Schema( - required(2, "id", Types.IntegerType.get(), "unique ID"), + required(2, "id", Types.IntegerType.get(), "unique ID 🤪"), required(3, "data", Types.StringType.get())); // another schema that is not the same