V2.7.2 release

src/ParquetViewer.Engine/Exceptions/UnsupportedFieldException.cs

@@ -2,7 +2,7 @@
 {
     public class UnsupportedFieldException : Exception
     {
-        public UnsupportedFieldException(string fieldName, Exception? ex = null) : base(fieldName, ex)
+        public UnsupportedFieldException(string message, Exception? ex = null) : base(message, ex)
         {
 
         }

src/ParquetViewer.Engine/ParquetEngine.Processor.cs

@@ -1,4 +1,5 @@
 using Parquet;
+using Parquet.Meta;
 using ParquetViewer.Engine.Exceptions;
 using System.Collections;
 using System.Data;
@@ -104,7 +105,7 @@ public async Task<DataTable> ReadRowsAsync(List<string> selectedFields, int offs
             }
         }
 
-        private static async Task ReadPrimitiveField(DataTable dataTable, ParquetRowGroupReader groupReader, int rowBeginIndex, ParquetSchemaElement field,
+        private async Task ReadPrimitiveField(DataTable dataTable, ParquetRowGroupReader groupReader, int rowBeginIndex, ParquetSchemaElement field,
             long skipRecords, long readRecords, bool isFirstColumn, Dictionary<int, DataRow> rowLookupCache, CancellationToken cancellationToken, IProgress<int>? progress)
         {
             int rowIndex = rowBeginIndex;
@@ -146,13 +147,58 @@ public async Task<DataTable> ReadRowsAsync(List<string> selectedFields, int offs
                     }
                 }
 
-                datarow[fieldIndex] = value ?? DBNull.Value;
+                datarow[fieldIndex] = FixDateTime(value, field) ?? DBNull.Value;
 
                 rowIndex++;
                 progress?.Report(1);
             }
         }
 
+        /// <summary>
+        /// This is a patch fix to handle malformed datetime fields. We assume TIMESTAMP fields are DateTime values.
+        /// </summary>
+        /// <param name="value">Original value</param>
+        /// <param name="field">Schema element</param>
+        /// <returns>If the field is a timestamp, a DateTime object will be returned. Otherwise the value will not be changed.</returns>
+        private object? FixDateTime(object value, ParquetSchemaElement field)
+        {
+            if (!this.FixMalformedDateTime || value is null)
+                return value;
+
+            var timestampSchema = field.SchemaElement?.LogicalType?.TIMESTAMP;
+            if (timestampSchema is not null && field.SchemaElement?.ConvertedType is null)
+            {
+                long castValue;
+                if (field.DataField?.ClrType == typeof(long?))
+                {
+                    castValue = ((long?)value).Value; //We know this isn't null from the null check above
+                }
+                else if (field.DataField?.ClrType == typeof(long))
+                {
+                    castValue = (long)value;
+                }
+                else
+                {
+                    throw new UnsupportedFieldException($"Field {field.Path} is not a valid timestamp field");
+                }
+
+                int divideBy = 0;
+                if (timestampSchema.Unit.NANOS != null)
+                    divideBy = 1000 * 1000;
+                else if (timestampSchema.Unit.MICROS != null)
+                    divideBy = 1000;
+                else if (timestampSchema.Unit.MILLIS != null)
+                    divideBy = 1;
+
+                if (divideBy > 0)
+                    value = DateTimeOffset.FromUnixTimeMilliseconds(castValue / divideBy).DateTime;
+                else //Not sure if this 'else' is correct but adding just in case
+                    value = DateTimeOffset.FromUnixTimeSeconds(castValue);
+            }
+
+            return value;
+        }
+
         private static async Task ReadListField(DataTable dataTable, ParquetRowGroupReader groupReader, int rowBeginIndex, ParquetSchemaElement field,
             long skipRecords, long readRecords, bool isFirstColumn, Dictionary<int, DataRow> rowLookupCache, CancellationToken cancellationToken, IProgress<int>? progress)
         {
@@ -162,7 +208,7 @@ public async Task<DataTable> ReadRowsAsync(List<string> selectedFields, int offs
             {
                 itemField = listField.GetChildOrSingle("item"); //Not all parquet files follow the same format so we're being lax with getting the child here
             }
-            catch(Exception ex)
+            catch (Exception ex)
             {
                 throw new UnsupportedFieldException($"Cannot load field '{field.Path}. Invalid List type.'", ex);
             }
@@ -312,14 +358,21 @@ private DataTable BuildDataTable(List<string> fields)
                 var schema = ParquetSchemaTree.GetChild(field);
 
                 DataColumn newColumn;
-                if (schema.SchemaElement.ConvertedType == Parquet.Meta.ConvertedType.LIST)
+                if (schema.SchemaElement.ConvertedType == ConvertedType.LIST)
                 {
                     newColumn = new DataColumn(field, typeof(ListValue));
                 }
-                else if (schema.SchemaElement.ConvertedType == Parquet.Meta.ConvertedType.MAP)
+                else if (schema.SchemaElement.ConvertedType == ConvertedType.MAP)
                 {
                     newColumn = new DataColumn(field, typeof(MapValue));
                 }
+                else if (this.FixMalformedDateTime
+                    && schema.SchemaElement.LogicalType?.TIMESTAMP is not null 
+                    && schema.SchemaElement?.ConvertedType is null)
+                {
+                    //Fix for malformed datetime fields (#88)
+                    newColumn = new DataColumn(field, typeof(DateTime));
+                }
                 else
                 {
                     var clrType = schema.DataField?.ClrType ?? throw new Exception($"{field} has no data field");

src/ParquetViewer.Engine/ParquetEngine.cs

@@ -29,6 +29,8 @@ public partial class ParquetEngine : IDisposable
 
         public string OpenFileOrFolderPath { get; }
 
+        public bool FixMalformedDateTime { get; set; } = true;
+
         private ParquetSchemaElement BuildParquetSchemaTree()
         {
             var thriftSchema = ThriftMetadata.Schema ?? throw new Exception("No thrift metadata was found");

src/ParquetViewer.Tests/ParquetViewer.Tests.csproj

@@ -59,6 +59,9 @@
     <None Update="Data\LIST_TYPE_TEST1.parquet">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="Data\MALFORMED_DATETIME_TEST1.parquet">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="Data\MAP_TYPE_TEST1.parquet">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

src/ParquetViewer.Tests/SanityTests.cs

@@ -243,5 +243,24 @@ public async Task NULLABLE_GUID_TEST1()
             Assert.Equal(new Guid("0cf9cbfd-d320-45d7-b29f-9c2de1baa979"), dataTable.Rows[0][1]);
             Assert.Equal(new DateTime(2019, 1, 1), dataTable.Rows[0][4]);
         }
+
+        [Fact]
+        public async Task MALFORMED_DATETIME_TEST1()
+        {
+            using var parquetEngine = await ParquetEngine.OpenFileOrFolderAsync("Data/MALFORMED_DATETIME_TEST1.parquet", default);
+
+            var dataTable = await parquetEngine.ReadRowsAsync(parquetEngine.Fields, 0, int.MaxValue, default);
+            Assert.Equal(typeof(DateTime), dataTable.Rows[0]["ds"]?.GetType());
+
+            //Check if the malformed datetime still needs to be fixed
+            parquetEngine.FixMalformedDateTime = false;
+
+            dataTable = await parquetEngine.ReadRowsAsync(parquetEngine.Fields, 0, int.MaxValue, default);
+            if (dataTable.Rows[0]["ds"]?.GetType() == typeof(DateTime))
+            {
+                Assert.Fail("Looks like the Malformed DateTime Fix is no longer needed! Remove that part of the code.");
+            }
+            Assert.Equal(typeof(long), dataTable.Rows[0]["ds"]?.GetType()); //If it's not a datetime, then it should be a long.
+        }
     }
 }