Merge pull request #2 from Seddryck/feature/Support_of_commentChar_(#1)

Implementation and testing of commentChar

PocketCsvReader.Testing/InternalCsvReaderTest.cs

@@ -30,14 +30,14 @@ public new string RemoveTextQualifier(string item, char textQualifier, char esca
             public new IEnumerable<string> SplitLine(string row, char fieldSeparator, char textQualifier, char escapeTextQualifier, string emptyCell)
                 => base.SplitLine(row, fieldSeparator, textQualifier, escapeTextQualifier, emptyCell);
 
-            public new int CountRecordSeparators(StreamReader reader, string recordSeparator, int bufferSize)
-                => base.CountRecordSeparators(reader, recordSeparator, bufferSize);
+            public new int CountRecordSeparators(StreamReader reader, string recordSeparator, char commentChar, int bufferSize)
+                => base.CountRecordSeparators(reader, recordSeparator, commentChar, bufferSize);
 
             public new string GetFirstRecord(StreamReader reader, string recordSeparator, int bufferSize)
                 => base.GetFirstRecord(reader, recordSeparator, bufferSize);
 
-            public new (IEnumerable<string>, string) GetNextRecords(StreamReader reader, string recordSeparator, int bufferSize, string alreadyRead)
-                => base.GetNextRecords(reader, recordSeparator, bufferSize, alreadyRead);
+            public new (IEnumerable<string>, string, bool) GetNextRecords(StreamReader reader, string recordSeparator, char commentChar, int bufferSize, string alreadyRead)
+                => base.GetNextRecords(reader, recordSeparator, commentChar, bufferSize, alreadyRead);
 
             public new bool IsLastRecord(string record)
                 => base.IsLastRecord(record);
@@ -172,7 +172,7 @@ public void CountRecordSeparator_Csv_CorrectCount(string text, string recordSepa
                 var reader = new CsvReaderProxy();
                 using (StreamReader streamReader = new StreamReader(stream, Encoding.UTF8, true))
                 {
-                    var value = reader.CountRecordSeparators(streamReader, recordSeparator, bufferSize);
+                    var value = reader.CountRecordSeparators(streamReader, recordSeparator, '#',  bufferSize);
                     Assert.That(value, Is.EqualTo(result));
                 }
                 writer.Dispose();
@@ -274,7 +274,7 @@ public void NextRecords_Csv_CorrectResults(string text, string recordSeparator,
                 var reader = new CsvReaderProxy();
                 using (var streamReader = new StreamReader(stream, Encoding.UTF8, true))
                 {
-                    var (values, extraRead) = reader.GetNextRecords(streamReader, recordSeparator, bufferSize, string.Empty);
+                    var (values, extraRead, eof) = reader.GetNextRecords(streamReader, recordSeparator, '#', bufferSize, string.Empty);
                     foreach (var value in values)
                     {
                         Assert.That(value, Does.StartWith("abc"));
@@ -297,7 +297,7 @@ public void NextRecords_CsvWithCsvProfileMissingCell_CorrectResults(string text,
 
                 stream.Position = 0;
                 var reader = new CsvReader();
-                var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', "_", missingCell);
+                var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', '#', "_", missingCell);
 
                 Assert.That(dataTable.Rows[0].ItemArray[0], Is.EqualTo("a"));
                 Assert.That(dataTable.Rows[0].ItemArray[1], Is.EqualTo("b"));
@@ -332,7 +332,7 @@ public void NextRecords_CsvWithCsvProfileEmptyCell_CorrectResults(string text, c
 
                 stream.Position = 0;
                 var reader = new CsvReader();
-                var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', emptyCell, "_");
+                var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', '#', emptyCell, "_");
 
                 Assert.That(dataTable.Rows[0].ItemArray[0], Is.EqualTo("a"));
                 Assert.That(dataTable.Rows[0].ItemArray[1], Is.EqualTo("b"));
@@ -518,5 +518,34 @@ public void Read_MissingValue_MatchWithNullValue()
                 }
             }
         }
+
+        [Test]
+        [TestCase("a;b;c\r\n1;2;3")]
+        [TestCase("a;b;c\r\n1;2;3\r\n")]
+        [TestCase("a;b;c\r\n#\r\n1;2;3")]
+        [TestCase("a;b;c\r\n#x;y;z\r\n1;2;3")]
+        [TestCase("a;b;c\r\n1;2;3\r\n#x;y;z")]
+        [TestCase("#x;y;z\r\na;b;c\r\n1;2;3")]
+        [TestCase("#x;y;z\r\n#x;y;z\r\na;b;c\r\n1;2;3")]
+        [TestCase("#x;y;z\r\n#x;y;z\r\na;b;c\r\n1;2;3\r\n#1;2;3")]
+        public void Read_Comment_CommentedLinesSkipped(string content)
+        {
+            using (var stream = new MemoryStream())
+            {
+                using (var writer = new StreamWriter(stream))
+                {
+                    writer.Write(content);
+                    writer.Flush();
+
+                    stream.Position = 0;
+
+                    var profile = new CsvProfile(new CsvDialectDescriptor { Header = false, Delimiter = ';', CommentChar='#', DoubleQuote=false });
+                    var reader = new CsvReaderProxy(profile);
+                    var dataTable = reader.Read(stream);
+                    Assert.That(dataTable.Rows.Count, Is.EqualTo(2));
+                    Assert.That(dataTable.Columns.Count, Is.EqualTo(3));
+                }
+            }
+        }
     }
 }

PocketCsvReader/CsvDialectDescriptor.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace PocketCsvReader
+{
+    public class CsvDialectDescriptor
+    {
+        public char Delimiter { get; set; } = ',';
+        public string LineTerminator { get; set; } = "\r\n";
+        public char QuoteChar { get; set; } = '"';
+        public bool DoubleQuote { get; set; } = true;
+        public char EscapeChar { get; set; }
+        public string NullSequence { get; set; }
+        public bool SkipInitialSpace { get; set; } = false;
+        public bool Header { get; set; } = true;
+        public char CommentChar { get; set; }
+        public bool CaseSensitiveHeader { get; set; } = false;
+        public string CsvDdfVersion { get; set; }
+    }
+}

PocketCsvReader/CsvProfile.cs

@@ -4,11 +4,7 @@ namespace PocketCsvReader
 {
     public class CsvProfile
     {
-        public virtual char FieldSeparator { get; private set; }
-        public virtual char TextQualifier { get; private set; }
-        public virtual char EscapeTextQualifier { get; private set; }
-        public virtual string RecordSeparator { get; private set; }
-        public virtual bool FirstRowHeader { get; private set; }
+        public CsvDialectDescriptor Descriptor { get; private set; }
         public virtual bool PerformanceOptmized { get; private set; }
         public virtual int BufferSize { get; private set; }
         public virtual string EmptyCell { get; private set; }
@@ -48,17 +44,39 @@ public CsvProfile(char fieldSeparator, char textQualifier, string recordSeparato
 
         public CsvProfile(char fieldSeparator, char textQualifier, char escapeTextQualifier, string recordSeparator, bool firstRowHeader, bool performanceOptimized, int bufferSize, string emptyCell, string missingCell)
         {
-            FieldSeparator = fieldSeparator;
-            TextQualifier = textQualifier;
-            EscapeTextQualifier = escapeTextQualifier;
-            RecordSeparator = recordSeparator;
-            FirstRowHeader = firstRowHeader;
+            Descriptor = new CsvDialectDescriptor
+            {
+                Delimiter = fieldSeparator,
+                QuoteChar = textQualifier,
+                EscapeChar = escapeTextQualifier,
+                LineTerminator = recordSeparator,
+                Header = firstRowHeader
+            };
+
             PerformanceOptmized = performanceOptimized;
             EmptyCell = emptyCell;
             MissingCell = missingCell;
             BufferSize = bufferSize;
         }
 
+        public CsvProfile(CsvDialectDescriptor descriptor)
+        {
+            if (descriptor.DoubleQuote)
+                throw new ArgumentException("PocketCsvReader doesn't support doubleQuote set to true in the CSV dialect descriptor.");
+            if (descriptor.NullSequence?.Length > 0)
+                throw new ArgumentException("PocketCsvReader doesn't support nullSequence set to any value in the CSV dialect descriptor.");
+            if (descriptor.SkipInitialSpace)
+                throw new ArgumentException("PocketCsvReader doesn't support skipInitialSpace set to true in the CSV dialect descriptor.");
+            if (descriptor.CaseSensitiveHeader)
+                throw new ArgumentException("PocketCsvReader doesn't support caseSensitiveHeader set to true in the CSV dialect descriptor.");
+
+            Descriptor = descriptor;
+            PerformanceOptmized = false;
+            EmptyCell = string.Empty;
+            MissingCell = string.Empty;
+            BufferSize = 4096;
+        }
+
         public static CsvProfile CommaDoubleQuote { get; } = new CsvProfile(',', '\"');
         public static CsvProfile SemiColumnDoubleQuote { get; } = new CsvProfile(';', '\"');
         public static CsvProfile TabDoubleQuote { get; } = new CsvProfile('\t', '\"');