Skip to content

Commit

Permalink
Fix #198: add CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS (#440)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cowtowncoder
cowtowncoder committed Oct 19, 2023
1 parent f8bd56b commit 90a2f8c
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 22 deletions.
9 changes: 9 additions & 0 deletions csv/src/main/java/com/fasterxml/jackson/dataformat/csv/CsvGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ public enum Feature
*/
ALWAYS_QUOTE_EMPTY_STRINGS(false),

/**
* Feature that determines whether values written as Nymbers (from {@code java.lang.Number}
* valued POJO properties) should be forced to be quoted, regardless of whether they
* actually need this.
*
* @since 2.16
*/
ALWAYS_QUOTE_NUMBERS(false),

/**
* Feature that determines whether quote characters within quoted String values are escaped
* using configured escape character, instead of being "doubled up" (that is: a quote character
Expand Down
78 changes: 64 additions & 14 deletions csv/src/main/java/com/fasterxml/jackson/dataformat/csv/impl/CsvEncoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ public class CsvEncoder

protected boolean _cfgAlwaysQuoteEmptyStrings;

// @since 2.16
protected boolean _cfgAlwaysQuoteNumbers;

protected boolean _cfgEscapeQuoteCharWithEscapeChar;

/**
Expand Down Expand Up @@ -218,6 +221,7 @@ public CsvEncoder(IOContext ctxt, int csvFeatures, Writer out, CsvSchema schema,
_cfgIncludeMissingTail = !CsvGenerator.Feature.OMIT_MISSING_TAIL_COLUMNS.enabledIn(_csvFeatures);
_cfgAlwaysQuoteStrings = CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS.enabledIn(csvFeatures);
_cfgAlwaysQuoteEmptyStrings = CsvGenerator.Feature.ALWAYS_QUOTE_EMPTY_STRINGS.enabledIn(csvFeatures);
_cfgAlwaysQuoteNumbers = CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS.enabledIn(csvFeatures);
_cfgEscapeQuoteCharWithEscapeChar = CsvGenerator.Feature.ESCAPE_QUOTE_CHAR_WITH_ESCAPE_CHAR.enabledIn(csvFeatures);
_cfgEscapeControlCharWithEscapeChar = Feature.ESCAPE_CONTROL_CHARS_WITH_ESCAPE_CHAR.enabledIn(csvFeatures);

Expand Down Expand Up @@ -257,6 +261,8 @@ public CsvEncoder(CsvEncoder base, CsvSchema newSchema)
_cfgIncludeMissingTail = base._cfgIncludeMissingTail;
_cfgAlwaysQuoteStrings = base._cfgAlwaysQuoteStrings;
_cfgAlwaysQuoteEmptyStrings = base._cfgAlwaysQuoteEmptyStrings;
_cfgAlwaysQuoteNumbers = base._cfgAlwaysQuoteNumbers;

_cfgEscapeQuoteCharWithEscapeChar = base._cfgEscapeQuoteCharWithEscapeChar;
_cfgEscapeControlCharWithEscapeChar = base._cfgEscapeControlCharWithEscapeChar;

Expand Down Expand Up @@ -329,6 +335,7 @@ public CsvEncoder overrideFormatFeatures(int feat) {
_cfgIncludeMissingTail = !CsvGenerator.Feature.OMIT_MISSING_TAIL_COLUMNS.enabledIn(feat);
_cfgAlwaysQuoteStrings = CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS.enabledIn(feat);
_cfgAlwaysQuoteEmptyStrings = CsvGenerator.Feature.ALWAYS_QUOTE_EMPTY_STRINGS.enabledIn(feat);
_cfgAlwaysQuoteNumbers = CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS.enabledIn(feat);
_cfgEscapeQuoteCharWithEscapeChar = CsvGenerator.Feature.ESCAPE_QUOTE_CHAR_WITH_ESCAPE_CHAR.enabledIn(feat);
_cfgEscapeControlCharWithEscapeChar = Feature.ESCAPE_CONTROL_CHARS_WITH_ESCAPE_CHAR.enabledIn(feat);
}
Expand Down Expand Up @@ -407,14 +414,20 @@ public final void write(int columnIndex, int value) throws IOException
// easy case: all in order
if (columnIndex == _nextColumnToWrite) {
// inlined 'appendValue(int)'
// up to 10 digits and possible minus sign, leading comma
if ((_outputTail + 12) > _outputEnd) {
// up to 10 digits and possible minus sign, leading comma, possible quotes
if ((_outputTail + 14) > _outputEnd) {
_flushBuffer();
}
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
_outputTail = NumberOutput.outputInt(value, _outputBuffer, _outputTail);
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
++_nextColumnToWrite;
return;
}
Expand All @@ -426,14 +439,20 @@ public final void write(int columnIndex, long value) throws IOException
// easy case: all in order
if (columnIndex == _nextColumnToWrite) {
// inlined 'appendValue(int)'
// up to 20 digits, minus sign, leading comma
if ((_outputTail + 22) > _outputEnd) {
// up to 20 digits, minus sign, leading comma, possible quotes
if ((_outputTail + 24) > _outputEnd) {
_flushBuffer();
}
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
_outputTail = NumberOutput.outputLong(value, _outputBuffer, _outputTail);
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
++_nextColumnToWrite;
return;
}
Expand Down Expand Up @@ -607,28 +626,40 @@ protected void appendRawValue(String value) throws IOException

protected void appendValue(int value) throws IOException
{
// up to 10 digits and possible minus sign, leading comma
if ((_outputTail + 12) > _outputEnd) {
// up to 10 digits and possible minus sign, leading comma, possible quotes
if ((_outputTail + 14) > _outputEnd) {
_flushBuffer();
}
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
_outputTail = NumberOutput.outputInt(value, _outputBuffer, _outputTail);
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
}

protected void appendValue(long value) throws IOException
{
// up to 20 digits, minus sign, leading comma
if ((_outputTail + 22) > _outputEnd) {
// up to 20 digits, minus sign, leading comma, possible quotes
if ((_outputTail + 24) > _outputEnd) {
_flushBuffer();
}
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
_outputTail = NumberOutput.outputLong(value, _outputBuffer, _outputTail);
if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
}
}

protected void appendValue(float value) throws IOException
{
String str = NumberOutput.toString(value, _cfgUseFastDoubleWriter);
Expand All @@ -639,7 +670,7 @@ protected void appendValue(float value) throws IOException
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
writeRaw(str);
writeNumber(str);
}

protected void appendValue(double value) throws IOException
Expand All @@ -652,11 +683,11 @@ protected void appendValue(double value) throws IOException
if (_nextColumnToWrite > 0) {
_outputBuffer[_outputTail++] = _cfgColumnSeparator;
}
writeRaw(str);
writeNumber(str);
}

// @since 2.16: pre-encoded BigInteger/BigDecimal value
protected void appendNumberValue(String numValue) throws IOException
protected void appendNumberValue(String numStr) throws IOException
{
// Same as "appendRawValue()", except may want quoting
if (_outputTail >= _outputEnd) {
Expand All @@ -665,7 +696,7 @@ protected void appendNumberValue(String numValue) throws IOException
if (_nextColumnToWrite > 0) {
appendColumnSeparator();
}
writeRaw(numValue);
writeNumber(numStr);
}

protected void appendValue(boolean value) throws IOException {
Expand Down Expand Up @@ -702,7 +733,7 @@ protected void appendColumnSeparator() throws IOException {
/* Output methods, unprocessed ("raw")
/**********************************************************
*/

public void writeRaw(String text) throws IOException
{
// Nothing to check, can just output as is
Expand Down Expand Up @@ -788,6 +819,25 @@ private void writeRawLong(String text) throws IOException
_outputTail = len;
}

// @since 2.16
private void writeNumber(String text) throws IOException
{
final int len = text.length();
if ((_outputTail + len + 2) > _outputEnd) {
_flushBuffer();
}

if (_cfgAlwaysQuoteNumbers) {
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
text.getChars(0, len, _outputBuffer, _outputTail);
_outputTail += len;
_outputBuffer[_outputTail++] = (char) _cfgQuoteCharacter;
} else {
text.getChars(0, len, _outputBuffer, _outputTail);
_outputTail += len;
}
}

/*
/**********************************************************
/* Output methods, with quoting and escaping
Expand Down
64 changes: 61 additions & 3 deletions csv/src/test/java/com/fasterxml/jackson/dataformat/csv/ser/CSVGeneratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.io.StringWriter;
import java.math.BigDecimal;
import java.math.BigInteger;

import com.fasterxml.jackson.annotation.JsonPropertyOrder;

Expand Down Expand Up @@ -51,6 +52,19 @@ public Entry3(String id, BigDecimal amount, boolean enabled) {
}
}

@JsonPropertyOrder({"id", "amount"})
static class NumberEntry<T> {
public String id;
public T amount;
public boolean enabled;

public NumberEntry(String id, T amount, boolean enabled) {
this.id = id;
this.amount = amount;
this.enabled = enabled;
}
}

/*
/**********************************************************************
/* Test methods
Expand Down Expand Up @@ -242,7 +256,7 @@ public void testForcedQuotingOfBigDecimal() throws Exception
.writeValueAsString(new Entry3("xyz", BigDecimal.valueOf(1.5), false));
assertEquals("xyz,1.5,false\n", result);
}

public void testForcedQuotingWithQuoteEscapedWithBackslash() throws Exception
{
CsvSchema schema = CsvSchema.builder()
Expand Down Expand Up @@ -370,12 +384,10 @@ public void testRawWrites() throws Exception
public void testSerializationOfPrimitivesToCsv() throws Exception
{
CsvMapper mapper = new CsvMapper();
/*
testSerializationOfPrimitiveToCsv(mapper, String.class, "hello world", "\"hello world\"\n");
testSerializationOfPrimitiveToCsv(mapper, Boolean.class, true, "true\n");
testSerializationOfPrimitiveToCsv(mapper, Integer.class, 42, "42\n");
testSerializationOfPrimitiveToCsv(mapper, Long.class, 42L, "42\n");
*/
testSerializationOfPrimitiveToCsv(mapper, Short.class, (short)42, "42\n");
testSerializationOfPrimitiveToCsv(mapper, Double.class, 42.33d, "42.33\n");
testSerializationOfPrimitiveToCsv(mapper, Float.class, 42.33f, "42.33\n");
Expand All @@ -390,6 +402,52 @@ private <T> void testSerializationOfPrimitiveToCsv(final CsvMapper mapper,
assertEquals(expectedCsv, csv);
}

// [dataformats-csv#198]: Verify quoting of Numbers
public void testForcedQuotingOfNumbers() throws Exception
{
final CsvSchema schema = CsvSchema.builder()
.addColumn("id")
.addColumn("amount")
.addColumn("enabled")
.build();
final CsvSchema reorderedSchema = CsvSchema.builder()
.addColumn("amount")
.addColumn("id")
.addColumn("enabled")
.build();
ObjectWriter w = MAPPER.writer(schema);
_testForcedQuotingOfNumbers(w, reorderedSchema,
new NumberEntry<Integer>("id", Integer.valueOf(42), true));
_testForcedQuotingOfNumbers(w, reorderedSchema,
new NumberEntry<Long>("id", Long.MAX_VALUE, false));
_testForcedQuotingOfNumbers(w, reorderedSchema,
new NumberEntry<BigInteger>("id", BigInteger.valueOf(-37), true));
_testForcedQuotingOfNumbers(w, reorderedSchema,
new NumberEntry<Double>("id", 2.25, false));
_testForcedQuotingOfNumbers(w, reorderedSchema,
new NumberEntry<BigDecimal>("id", BigDecimal.valueOf(-10.5), true));
}

private void _testForcedQuotingOfNumbers(ObjectWriter w, CsvSchema reorderedSchema,
NumberEntry<?> bean) throws Exception
{
// First verify with quoting
ObjectWriter w2 = w.with(CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS);
assertEquals(String.format("%s,\"%s\",%s\n", bean.id, bean.amount, bean.enabled),
w2.writeValueAsString(bean));

// And then dynamically disabled variant
ObjectWriter w3 = w2.without(CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS);
assertEquals(String.format("%s,%s,%s\n", bean.id, bean.amount, bean.enabled),
w3.writeValueAsString(bean));

// And then quoted but reordered to force buffering
ObjectWriter w4 = MAPPER.writer(reorderedSchema)
.with(CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS);
assertEquals(String.format("\"%s\",%s,%s\n", bean.amount, bean.id, bean.enabled),
w4.writeValueAsString(bean));
}

/*
/**********************************************************************
/* Secondary test methods
Expand Down
13 changes: 8 additions & 5 deletions csv/src/test/java/com/fasterxml/jackson/dataformat/csv/ser/SchemaReorderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,12 @@ public class SchemaReorderTest extends ModuleTestBase
{
// should work ok since CsvMapper forces alphabetic ordering as default:
static class Reordered {
public int a, b, c, d;
public int a;
public long b;
public long c;
public int d;
}

private final CsvMapper MAPPER = new CsvMapper();

public void testSchemaWithOrdering() throws Exception
Expand All @@ -24,13 +27,13 @@ public void testSchemaWithOrdering() throws Exception

Reordered value = new Reordered();
value.a = 1;
value.b = 2;
value.c = 3;
value.b = Long.MIN_VALUE;
value.c = Long.MAX_VALUE;
value.d = 4;

schema = schema.withHeader();
String csv = MAPPER.writer(schema).writeValueAsString(Arrays.asList(value));
assertEquals("b,c,a,d\n2,3,1,4\n", csv);
assertEquals("b,c,a,d\n"+Long.MIN_VALUE+","+Long.MAX_VALUE+",1,4\n", csv);

// _verifyLinks(schema);
}
Expand Down
2 changes: 2 additions & 0 deletions release-notes/VERSION-2.x
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Active Maintainers:

2.16.0 (not yet released)

#198: (csv) Support writing numbers as quoted Strings with
`CsvGenerator.Feature.ALWAYS_QUOTE_NUMBERS`
#422: (csv) Add `removeColumn()` method in `CsvSchema.Builder`
#435: (yaml) Minor parsing validation miss: tagged as `int`, exception
on underscore-only values
Expand Down

0 comments on commit 90a2f8c

Please sign in to comment.