[fix] Fix comments

apache · Jul 30, 2021 · 6058577 · 6058577
1 parent 14ae506
commit 6058577
Showing 1 changed file with 70 additions and 0 deletions.
diff --git a/...ormats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java b/...ormats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
@@ -18,10 +18,14 @@
 
 package org.apache.flink.formats.csv;
 
+import org.apache.flink.table.api.config.ExecutionConfigOptions;
 import org.apache.flink.table.planner.runtime.batch.sql.BatchFileSystemITCaseBase;
+import org.apache.flink.table.planner.runtime.utils.BatchTestBase;
 import org.apache.flink.types.Row;
+import org.apache.flink.util.CollectionUtil;
 import org.apache.flink.util.FileUtils;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.runners.Enclosed;
 import org.junit.runner.RunWith;
@@ -90,4 +94,70 @@ public void testEscapeChar() throws Exception {
                     Arrays.asList(Row.of("x5,5,1,1"), Row.of("x5,5,2,2")));
         }
     }
+
+    /**
+     * IT case which checks for a bug in Jackson 2.10. When the 4000th character in csv file is the
+     * new line character (\n) an exception will be thrown. After upgrading jackson to >= 2.11 this
+     * bug should not exist anymore.
+     */
+    public static class JacksonVersionUpgradeITCase extends BatchTestBase {
+
+        @Test
+        public void testCsvFileWithNewLineAt4000() throws Exception {
+            StringBuilder csvContent = new StringBuilder("# ");
+            for (int i = 0; i < 97; i++) {
+                csvContent.append("-");
+            }
+            csvContent.append("\n");
+            for (int i = 0; i < 50; i++) {
+                for (int j = 0; j < 49; j++) {
+                    csvContent.append("a");
+                }
+                csvContent.append(",");
+                for (int j = 0; j < 49; j++) {
+                    csvContent.append("b");
+                }
+                csvContent.append("\n");
+            }
+
+            File tempCsvFile = File.createTempFile("new-line-at-4000", ".csv");
+            tempCsvFile.createNewFile();
+            FileUtils.writeFileUtf8(tempCsvFile, csvContent.toString());
+
+            tEnv().getConfig()
+                    .getConfiguration()
+                    .setInteger(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);
+            tEnv().executeSql(
+                            "CREATE TABLE T (\n"
+                                    + "  a VARCHAR,\n"
+                                    + "  b VARCHAR\n"
+                                    + ") WITH (\n"
+                                    + "  'connector' = 'filesystem',\n"
+                                    + "  'path' = 'file://"
+                                    + tempCsvFile.toString()
+                                    + "',\n"
+                                    + "  'format' = 'csv',\n"
+                                    + "  'csv.allow-comments' = 'true'\n"
+                                    + ")")
+                    .await();
+            List<Row> results =
+                    CollectionUtil.iteratorToList(
+                            tEnv().executeSql("SELECT a, b FROM T").collect());
+
+            Assert.assertEquals(50, results.size());
+            for (Row actual : results) {
+                StringBuilder a = new StringBuilder();
+                for (int i = 0; i < 49; i++) {
+                    a.append("a");
+                }
+                StringBuilder b = new StringBuilder();
+                for (int i = 0; i < 49; i++) {
+                    b.append("b");
+                }
+                Assert.assertEquals(2, actual.getArity());
+                Assert.assertEquals(a.toString(), actual.getField(0));
+                Assert.assertEquals(b.toString(), actual.getField(1));
+            }
+        }
+    }
 }