[HUDI-5545] Extending support to other special characters (apache#7585)

Co-authored-by: sivabalan <n.siva.b@gmail.com>
fengjian428 · Jan 31, 2023 · 59610bc · 59610bc
1 parent 1da9220
commit 59610bc
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/...ilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java b/...ilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java
@@ -151,7 +151,9 @@ public Pair<List<String>, String> getNextEventsFromQueue(AmazonSQS sqs,
               .getTime()).max().orElse(lastCheckpointStr.map(Long::parseLong).orElse(0L));
 
       for (Map<String, Object> eventRecord : eventRecords) {
-        filteredEventRecords.add(new ObjectMapper().writeValueAsString(eventRecord).replace("%3D", "="));
+        filteredEventRecords.add(new ObjectMapper().writeValueAsString(eventRecord).replace("%3D", "=")
+            .replace("%24", "$").replace("%A3", "£").replace("%23", "#").replace("%26", "&").replace("%3F", "?")
+            .replace("%7E", "~").replace("%25", "%").replace("%2B", "+"));
       }
       // Return the old checkpoint if no messages to consume from queue.
       String newCheckpoint = newCheckpointTime == 0 ? lastCheckpointStr.orElse(null) : String.valueOf(newCheckpointTime);

diff --git a/...ies/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java b/...ies/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java
@@ -89,7 +89,8 @@ public void testNextEventsFromQueueShouldReturnsEventsFromQueue(Class<?> clazz)
     S3EventsMetaSelector selector = (S3EventsMetaSelector) ReflectionUtils.loadClass(clazz.getName(), props);
     // setup s3 record
     String bucket = "test-bucket";
-    String key = "part-foo-bar.snappy.parquet";
+    String key = "part%3Dpart%2Bpart%24part%A3part%23part%26part%3Fpart%7Epart%25.snappy.parquet";
+    String keyRes = "part=part+part$part£part#part&part?part~part%.snappy.parquet";
     Path path = new Path(bucket, key);
     CloudObjectTestUtils.setMessagesInQueue(sqs, path);
 
@@ -102,7 +103,7 @@ public void testNextEventsFromQueueShouldReturnsEventsFromQueue(Class<?> clazz)
     assertEquals(1, eventFromQueue.getLeft().size());
     assertEquals(1, processed.size());
     assertEquals(
-        key,
+        keyRes,
         new JSONObject(eventFromQueue.getLeft().get(0))
             .getJSONObject("s3")
             .getJSONObject("object")