Skip to content

Commit

Permalink
Add support for multi-character ValueSeparators (#12)
Browse files Browse the repository at this point in the history 
squashes 19 commits; implements read and write support for multi-character value separators per #10 (comment)
  • Loading branch information
@kevin-montrose
kevin-montrose committed Jun 7, 2020
1 parent 2c344e9 commit e64f7cf
Show file tree
Hide file tree
Showing 37 changed files with 3,430 additions and 459 deletions.
4 changes: 2 additions & 2 deletions Cesil.Benchmark/Benchmarks/Internals/NeedsEncodeBenchmark.cs
View file
Expand Up @@ -18,7 +18,7 @@ public class NeedsEncodeBenchmark

private List<string> Strings;

private char BasicContains_C1;
private string BasicContains_C1;
private char? BasicContains_C2;
private char? BasicContains_C3;

Expand All @@ -40,7 +40,7 @@ public unsafe void Initialize()

Strings = strRet;

BasicContains_C1 = ',';
BasicContains_C1 = ",";
BasicContains_C2 = '"';
BasicContains_C3 = null;

Expand Down
12 changes: 6 additions & 6 deletions Cesil.Tests/BufferWithPushbackTests.cs
View file
Expand Up @@ -23,7 +23,7 @@ public void Read()
MemoryPool<char>.Shared,
500
);
var bytes = buf.Read(new TextReaderAdapter(str));
var bytes = buf.Read(new TextReaderAdapter(str), true);
Assert.Equal(TEXT.Length, bytes);

var shouldMatch = new string(buf.Buffer.Span.Slice(0, bytes));
Expand All @@ -50,7 +50,7 @@ await using (var str = await getReader(TEXT))
MemoryPool<char>.Shared,
500
);
var bytes = await buf.ReadAsync(str, CancellationToken.None);
var bytes = await buf.ReadAsync(str, true, CancellationToken.None);
Assert.Equal(TEXT.Length, bytes);
var shouldMatch = new string(buf.Buffer.Span.Slice(0, bytes));
Expand All @@ -76,15 +76,15 @@ public void ReadAndPushback()
MemoryPool<char>.Shared,
500
);
var bytes = buf.Read(new TextReaderAdapter(str));
var bytes = buf.Read(new TextReaderAdapter(str), true);
Assert.Equal(TEXT.Length, bytes);

var shouldMatch = new string(buf.Buffer.Span.Slice(0, bytes));
Assert.Equal(TEXT, shouldMatch);

buf.PushBackFromBuffer(bytes, SECOND_TEXT.Length);

var bytes2 = buf.Read(new TextReaderAdapter(str));
var bytes2 = buf.Read(new TextReaderAdapter(str), true);

Assert.Equal(SECOND_TEXT.Length, bytes2);

Expand All @@ -111,15 +111,15 @@ await using (var str = await getReader(TEXT))
MemoryPool<char>.Shared,
500
);
var bytes = await buf.ReadAsync(str, CancellationToken.None);
var bytes = await buf.ReadAsync(str, true, CancellationToken.None);
Assert.Equal(TEXT.Length, bytes);
var shouldMatch = new string(buf.Buffer.Span.Slice(0, bytes));
Assert.Equal(TEXT, shouldMatch);
buf.PushBackFromBuffer(bytes, SECOND_TEXT.Length);
var bytes2 = await buf.ReadAsync(str, CancellationToken.None);
var bytes2 = await buf.ReadAsync(str, true, CancellationToken.None);
Assert.Equal(SECOND_TEXT.Length, bytes2);
Expand Down
96 changes: 87 additions & 9 deletions Cesil.Tests/ConfigurationTests.cs
View file
Expand Up @@ -181,15 +181,15 @@ public void CharLookupInBounds()
var cOpts =
Options.CreateBuilder(Options.Default)
.WithEscapedValueStartAndEnd(char.MinValue)
.WithValueSeparator(char.MaxValue)
.WithValueSeparator(char.MaxValue.ToString())
.WithEscapedValueEscapeCharacter('c')
.WithCommentCharacter('d')
.ToOptions();

var dOpts =
Options.CreateBuilder(Options.Default)
.WithEscapedValueStartAndEnd('"')
.WithValueSeparator(',')
.WithValueSeparator(','.ToString())
.WithEscapedValueEscapeCharacter('"')
.WithCommentCharacter('#')
.ToOptions();
Expand Down Expand Up @@ -294,7 +294,7 @@ public void OptionsEquality()
.WithReadHeader(rh)
.WithRowEnding(re)
.WithTypeDescriber(typeDesc)
.WithValueSeparator(valSepChar)
.WithValueSeparator(valSepChar.ToString())
.WithWriteBufferSizeHint(writeHint)
.WithWriteHeader(wh)
.WithWriteTrailingRowEnding(wt)
Expand Down Expand Up @@ -457,9 +457,15 @@ private static void _SingleColumn<T>(string n)
[Fact]
public void OptionsValidation()
{
Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(',').WithEscapedValueStartAndEnd(',').ToOptions());
Assert.Throws<ArgumentNullException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(null));
Assert.Throws<ArgumentException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(""));

Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(',').WithCommentCharacter(',').ToOptions());
Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparatorInternal(null).ToOptions());
Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparatorInternal("").ToOptions());

Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(','.ToString()).WithEscapedValueStartAndEnd(',').ToOptions());

Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithValueSeparator(','.ToString()).WithCommentCharacter(',').ToOptions());

Assert.Throws<InvalidOperationException>(() => Options.CreateBuilder(Options.Default).WithEscapedValueStartAndEnd(',').WithCommentCharacter(',').ToOptions());

Expand All @@ -474,7 +480,7 @@ public void OptionsValidation()

Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder().WithValueSeparator(',')
Options.CreateBuilder().WithValueSeparator(','.ToString())
.WithRowEnding(RowEnding.CarriageReturnLineFeed)
.WithEscapedValueStartAndEnd('"')
.WithEscapedValueEscapeCharacter('"')
Expand All @@ -494,7 +500,7 @@ public void OptionsValidation()

Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder().WithValueSeparator(',')
Options.CreateBuilder().WithValueSeparator(','.ToString())
.WithRowEnding(RowEnding.CarriageReturnLineFeed)
.WithEscapedValueStartAndEnd('"')
.WithEscapedValueEscapeCharacter('"')
Expand Down Expand Up @@ -569,15 +575,15 @@ public void OptionsValidation()
() =>
Options.CreateBuilder(Options.Default)
.WithWhitespaceTreatmentInternal(WhitespaceTreatments.Trim)
.WithValueSeparator(' ')
.WithValueSeparator(' '.ToString())
.ToOptions()
);

Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithExtraColumnTreatmentInternal(0)
.WithValueSeparator(' ')
.WithValueSeparator(' '.ToString())
.ToOptions()
);

Expand All @@ -588,6 +594,78 @@ public void OptionsValidation()
.WithEscapedValueEscapeCharacter('\\')
.ToOptions()
);

// \r clashing
{
Options.CreateBuilder(Options.Default).WithValueSeparator("\r").WithRowEnding(RowEnding.LineFeed).ToOptions();
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r")
.WithRowEnding(RowEnding.CarriageReturn)
.ToOptions()
);
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r")
.WithRowEnding(RowEnding.CarriageReturnLineFeed)
.ToOptions()
);
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r")
.WithRowEnding(RowEnding.Detect)
.ToOptions()
);
}

// \n clashing
{
Options.CreateBuilder(Options.Default).WithValueSeparator("\n").WithRowEnding(RowEnding.CarriageReturn).ToOptions();
Options.CreateBuilder(Options.Default).WithValueSeparator("\n").WithRowEnding(RowEnding.CarriageReturnLineFeed).ToOptions();
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\n")
.WithRowEnding(RowEnding.LineFeed)
.ToOptions()
);
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\n")
.WithRowEnding(RowEnding.Detect)
.ToOptions()
);
}

// \r\n clashing
{
Options.CreateBuilder(Options.Default).WithValueSeparator("\n").WithRowEnding(RowEnding.CarriageReturnLineFeed).ToOptions();
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r\n")
.WithRowEnding(RowEnding.CarriageReturnLineFeed)
.ToOptions()
);
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r")
.WithRowEnding(RowEnding.CarriageReturnLineFeed)
.ToOptions()
);
Assert.Throws<InvalidOperationException>(
() =>
Options.CreateBuilder(Options.Default)
.WithValueSeparator("\r\n")
.WithRowEnding(RowEnding.Detect)
.ToOptions()
);
}
}

private class _BadCreateCalls
Expand Down
3 changes: 2 additions & 1 deletion Cesil.Tests/DisposalTests.cs
View file
Expand Up @@ -1114,7 +1114,8 @@ static RowEndingDetector MakeDetector()
new ReaderStateMachine(),
Options.Default,
CharacterLookup.MakeCharacterLookup(Options.Default, out _),
new TextReaderAdapter(TextReader.Null)
new TextReaderAdapter(TextReader.Null),
Options.Default.ValueSeparator.AsMemory()
);
}
}
Expand Down

0 comments on commit e64f7cf

Please sign in to comment.