From 97ae7f8d7f5c03bf63ebcc7fee74780f9abf7954 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Fri, 19 Oct 2018 10:44:00 +0200 Subject: [PATCH 1/2] NIFI-5525 - CSVRecordReader fails with StringIndexOutOfBoundsException when field is a double quote --- .../org/apache/nifi/csv/AbstractCSVRecordReader.java | 9 ++++++--- .../java/org/apache/nifi/csv/TestCSVRecordReader.java | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java index 62ddc8142fff..7642ae5b4b9a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java @@ -17,7 +17,6 @@ package org.apache.nifi.csv; - import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.serialization.RecordReader; import org.apache.nifi.serialization.record.DataType; @@ -79,7 +78,7 @@ protected final Object convert(final String value, final DataType dataType, fina return value; } - final String trimmed = value.startsWith("\"") && value.endsWith("\"") && (value.length() > 1) ? value.substring(1, value.length() - 1) : value; + final String trimmed = trim(value); if (trimmed.isEmpty()) { return null; } @@ -92,7 +91,7 @@ protected final Object convertSimpleIfPossible(final String value, final DataTyp return value; } - final String trimmed = value.startsWith("\"") && value.endsWith("\"") ? value.substring(1, value.length() - 1) : value; + final String trimmed = trim(value); if (trimmed.isEmpty()) { return null; } @@ -132,6 +131,10 @@ protected final Object convertSimpleIfPossible(final String value, final DataTyp return value; } + private String trim(String value) { + return value.startsWith("\"") && value.endsWith("\"") && (value.length() > 1) ? value.substring(1, value.length() - 1) : value; + } + @Override public RecordSchema getSchema() { return schema; diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java index 5095767afbde..7ddcc29bb05c 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java @@ -597,7 +597,7 @@ public void testMultipleRecordsEscapedWithSpecialChar() throws IOException, Malf @Test public void testQuote() throws IOException, MalformedRecordException { final CSVFormat format = CSVFormat.RFC4180.withFirstRecordAsHeader().withTrim().withQuote('"'); - final String text = "\"name\"\n\"\"\"\""; + final String text = "\"name\"\n\"\"\"\"\n\"\"\"\""; final List fields = new ArrayList<>(); fields.add(new RecordField("name", RecordFieldType.STRING.getDataType())); @@ -607,9 +607,12 @@ public void testQuote() throws IOException, MalformedRecordException { final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), StandardCharsets.UTF_8.name())) { - final Record record = reader.nextRecord(); - final String name = (String)record.getValue("name"); + Record record = reader.nextRecord(); + String name = (String)record.getValue("name"); + assertEquals("\"", name); + record = reader.nextRecord(false, false); + name = (String)record.getValue("name"); assertEquals("\"", name); } } From 2a30be68d7b483bd69aad6eca9ea0f03ec974e26 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 22 Oct 2018 18:31:36 +0200 Subject: [PATCH 2/2] review --- .../main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java index 7642ae5b4b9a..746b1ce858b8 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/AbstractCSVRecordReader.java @@ -132,7 +132,7 @@ protected final Object convertSimpleIfPossible(final String value, final DataTyp } private String trim(String value) { - return value.startsWith("\"") && value.endsWith("\"") && (value.length() > 1) ? value.substring(1, value.length() - 1) : value; + return (value.length() > 1) && value.startsWith("\"") && value.endsWith("\"") ? value.substring(1, value.length() - 1) : value; } @Override