From 997858dc0f0f7f5c628ff72007ba8768d9a425fc Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 16 Apr 2020 21:36:24 -0700 Subject: [PATCH] Fix #174 (or more precisely, refactor test that shows earlier fix) --- .../csv/deser/SkipEmptyLines191Test.java | 90 ++++++++++++++++++- .../csv/failing/SkipEmptyLines174Test.java | 83 ----------------- release-notes/CREDITS-2.x | 8 +- release-notes/VERSION-2.x | 2 + 4 files changed, 95 insertions(+), 88 deletions(-) delete mode 100644 csv/src/test/java/com/fasterxml/jackson/dataformat/csv/failing/SkipEmptyLines174Test.java diff --git a/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines191Test.java b/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines191Test.java index f0b4786b..1a7a3da1 100644 --- a/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines191Test.java +++ b/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines191Test.java @@ -2,22 +2,106 @@ import java.io.Reader; import java.io.StringReader; +import java.io.StringWriter; import java.util.List; import java.util.Map; +import java.util.Random; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import com.fasterxml.jackson.dataformat.csv.ModuleTestBase; // [dataformats-text#191] +// [dataformats-text#174] public class SkipEmptyLines191Test extends ModuleTestBase { - private static String COL_1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - private static String COL_2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + // [dataformats-text#174] + @JsonPropertyOrder({ "timestamp", "random" }) + static class Row174 { + private int timestamp; + private String random; + + public int getTimestamp() { + return timestamp; + } + + public void setTimestamp(int timestamp) { + this.timestamp = timestamp; + } + + public String getRandom() { + return random; + } + + public void setRandom(String random) { + this.random = random; + } + + @Override + public String toString() { + return "Row{timestamp=" + timestamp + ", random='" + random + "'}"; + } + } + + /* + /********************************************************************** + /* Test methods + /********************************************************************** + */ + + private final static CsvMapper MAPPER = new CsvMapper(); + + // [dataformats-text#174] + public void testEmptyLines174() throws Exception + { + final StringWriter sw = new StringWriter(50000); + int lineCount = 0; + final Random rnd = new Random(); + + while (lineCount < 4000) { + sw.append("\"" + System.currentTimeMillis()/1000 + "\",\"" + randomString(rnd) + "\"\n"); + ++lineCount; + } + final String doc = sw.toString(); + + ObjectReader objectReader = MAPPER + .enable(CsvParser.Feature.SKIP_EMPTY_LINES) + .readerFor(Row174.class) + .with(MAPPER.schemaFor(Row174.class)); + + MappingIterator iterator = objectReader.readValues(doc); + Row174 data = null; + lineCount = 0; + while (iterator.hasNext()) { + ++lineCount; + try { + data = iterator.next(); + } catch (Exception e) { + fail("Failed on row #"+lineCount+", previous row: "+data); + } + } + iterator.close(); + } + + private String randomString(Random rnd) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 10; ++i) { + sb.append((char) ('A' + (rnd.nextInt() & 0xF))); + } + return sb.toString(); + } // [dataformats-text#191]: IndexArrayOutOfBounds at 4000 public void testBigCsvFile() throws Exception { + final String COL_1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + final String COL_2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; CsvSchema schema = CsvSchema .emptySchema() .withHeader() @@ -26,7 +110,7 @@ public void testBigCsvFile() throws Exception .withComments(); try (Reader r = new StringReader(_generate4kDoc())) { - List> result = new CsvMapper() + List> result = MAPPER .readerFor(Map.class) .with(schema) .>readValues(r) diff --git a/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/failing/SkipEmptyLines174Test.java b/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/failing/SkipEmptyLines174Test.java deleted file mode 100644 index 646f9532..00000000 --- a/csv/src/test/java/com/fasterxml/jackson/dataformat/csv/failing/SkipEmptyLines174Test.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.fasterxml.jackson.dataformat.csv.failing; - -import java.io.*; -import java.util.Random; - -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.dataformat.csv.*; - -public class SkipEmptyLines174Test extends ModuleTestBase -{ - private final static CsvMapper MAPPER = new CsvMapper(); - - @JsonPropertyOrder({ "timestamp", "random" }) - static class Row { - private int timestamp; - private String random; - - public int getTimestamp() { - return timestamp; - } - - public void setTimestamp(int timestamp) { - this.timestamp = timestamp; - } - - public String getRandom() { - return random; - } - - public void setRandom(String random) { - this.random = random; - } - - @Override - public String toString() { - return "Row{timestamp=" + timestamp + ", random='" + random + "'}"; - } - } - - public void testEmptyLines174() throws Exception - { - String doc = generateCsvFile(); - ObjectReader objectReader = MAPPER - .enable(CsvParser.Feature.SKIP_EMPTY_LINES) - .readerFor(Row.class) - .with(MAPPER.schemaFor(Row.class)); - int lineCount = 0; - MappingIterator iterator = objectReader.readValues(doc); - Row data = null; - while (iterator.hasNext()) { - ++lineCount; - try { - data = iterator.next(); - } catch (Exception e) { -// System.out.println(lineCount + " : " + data); - fail("Failed on row #"+lineCount+", previous row: "+data); - } - } - iterator.close(); - } - - private String generateCsvFile() throws Exception { - final StringWriter sw = new StringWriter(50000); - int lineCount = 0; - final Random rnd = new Random(); - - while (lineCount < 4000) { - sw.append("\"" + System.currentTimeMillis()/1000 + "\",\"" + randomString(rnd) + "\"\n"); - ++lineCount; - } - return sw.toString(); - } - - private String randomString(Random rnd) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 10; ++i) { - sb.append((char) ('A' + (rnd.nextInt() & 0xF))); - } - return sb.toString(); - } -} diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 541a0351..835e069a 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -23,7 +23,7 @@ Fabrice Delhoste (spifd@github) Thomas Hauk (thauk-copperleaf@github) -* Contibuted #84 (yaml): Add option to allow use of platform-linefeed +* Contributed #84 (yaml): Add option to allow use of platform-linefeed (`YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS`) (2.9.6) @@ -103,7 +103,6 @@ Jochen Schalanda (joschi@github) * Reported #187: Update to SnakeYAML to 1.26 (from 1.24) to address CVE-2017-18640 (2.10.4) - Tyler Carpenter-Rivers (tyler2cr@github) #7: Add `CsvParser.Feature.EMPTY_STRING_AS_NULL` to allow coercing empty Strings into `null` values @@ -112,3 +111,8 @@ Tyler Carpenter-Rivers (tyler2cr@github) * Reported, constributed fix for #180: (yaml) YAMLGenerator serializes string with special chars unquoted when using `MINIMIZE_QUOTES` mode (2.11.0) + +Yohann BONILLO (ybonillo@github) + +* Reported #174: (csv) `CsvParser.Feature.SKIP_EMPTY_LINES` results in a mapping error + (2.11.0) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 77c3157b..9ee9e182 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -15,6 +15,8 @@ Modules: (contributed by Tyler C-R) #115: (csv) JsonProperty index is not honored by CsvSchema builder -- actually fixed by [databind#2555] +#174: (csv) `CsvParser.Feature.SKIP_EMPTY_LINES` results in a mapping error + (reported by Yohann B) #180: (yaml) YAMLGenerator serializes string with special chars unquoted when using `MINIMIZE_QUOTES` mode (reported, fix contributed by Timo R)