Migrate appender implementation to data chunks.

Giorgi · Apr 22, 2024 · 69e7119 · 69e7119
1 parent 01f0534
commit 69e7119
Show file tree

Hide file tree

Showing 9 changed files with 143 additions and 81 deletions.
diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs
@@ -56,7 +56,7 @@ public DuckDBAppenderRow CreateRow()
         }
 
         rowCount++;
-        return new DuckDBAppenderRow(nativeAppender, qualifiedTableName, vectorWriters, rowCount - 1);
+        return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowCount - 1);
     }
 
     public void Close()
@@ -99,9 +99,8 @@ private unsafe void InitVectorWriters()
         for (long index = 0; index < vectorWriters.LongLength; index++)
         {
             var vector = NativeMethods.DataChunks.DuckDBDataChunkGetVector(dataChunk, index);
-            var vectorData = NativeMethods.Vectors.DuckDBVectorGetData(vector);
 
-            vectorWriters[index] = VectorDataWriterFactory.CreateWriter(vector, vectorData, logicalTypes[index]);
+            vectorWriters[index] = VectorDataWriterFactory.CreateWriter(vector, logicalTypes[index]);
         }
     }
 

diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs
@@ -8,14 +8,12 @@ namespace DuckDB.NET.Data;
 public class DuckDBAppenderRow
 {
     private int columnIndex = 0;
-    private readonly Native.DuckDBAppender appender;
     private readonly string qualifiedTableName;
     private readonly VectorDataWriterBase[] vectorWriters;
     private readonly ulong rowIndex;
 
-    internal DuckDBAppenderRow(Native.DuckDBAppender appender, string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex)
+    internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex)
     {
-        this.appender = appender;
         this.qualifiedTableName = qualifiedTableName;
         this.vectorWriters = vectorWriters;
         this.rowIndex = rowIndex;
@@ -29,9 +27,9 @@ public void EndRow()
         }
     }
 
-    public DuckDBAppenderRow AppendNullValue() => Append<int>(null); //Doesn't matter what type T we pass to Append when passing null.
+    public DuckDBAppenderRow AppendNullValue() => AppendValueInternal<int?>(null); //Doesn't matter what type T we pass to Append when passing null.
 
-    public DuckDBAppenderRow AppendValue(bool? value) => AppendValue2(value);
+    public DuckDBAppenderRow AppendValue(bool? value) => AppendValueInternal(value);
 
 #if NET6_0_OR_GREATER
 
@@ -40,84 +38,68 @@ public void EndRow()
     public DuckDBAppenderRow AppendValue(Span<byte> value) => AppendSpan(value);
 #endif
 
-    public DuckDBAppenderRow AppendValue(string? value) => AppendValue2(value);
+    public DuckDBAppenderRow AppendValue(string? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(decimal? value) => AppendValue2(value);
+    public DuckDBAppenderRow AppendValue(decimal? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(Guid? value) => AppendValue2(value);
+    public DuckDBAppenderRow AppendValue(Guid? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(BigInteger? value, bool unsigned = false)
-    {
-        if (value == null)
-        {
-            return AppendNullValue();
-        }
-
-        if (unsigned)
-        {
-            Append<DuckDBUHugeInt>(new DuckDBUHugeInt(value.Value));
-        }
-        else
-        {
-            Append<DuckDBHugeInt>(new DuckDBHugeInt(value.Value));
-        }
-
-        return this;
-    }
+    public DuckDBAppenderRow AppendValue(BigInteger? value) => AppendValueInternal(value);
 
     #region Append Signed Int
 
-    public DuckDBAppenderRow AppendValue(sbyte? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(sbyte? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(short? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(short? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(int? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(int? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(long? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(long? value) => AppendValueInternal(value);
 
     #endregion
 
     #region Append Unsigned Int
 
-    public DuckDBAppenderRow AppendValue(byte? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(byte? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(ushort? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(ushort? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(uint? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(uint? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(ulong? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(ulong? value) => AppendValueInternal(value);
 
     #endregion
 
     #region Append Float
 
-    public DuckDBAppenderRow AppendValue(float? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(float? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(double? value) => Append(value);
+    public DuckDBAppenderRow AppendValue(double? value) => AppendValueInternal(value);
 
     #endregion
 
     #region Append Temporal
 #if NET6_0_OR_GREATER
-    public DuckDBAppenderRow AppendValue(DateOnly? value) => Append(value == null ? (DuckDBDate?)null : NativeMethods.DateTimeHelpers.DuckDBToDate(value.Value));
+    public DuckDBAppenderRow AppendValue(DateOnly? value) => AppendValueInternal(value);
 
-    public DuckDBAppenderRow AppendValue(TimeOnly? value) => Append(value == null ? (DuckDBTime?)null : NativeMethods.DateTimeHelpers.DuckDBToTime(value.Value));
-#else
-    public DuckDBAppenderRow AppendValue(DuckDBDateOnly? value) => Append(value == null ? (DuckDBDate?)null : NativeMethods.DateTimeHelpers.DuckDBToDate(value.Value));
-
-    public DuckDBAppenderRow AppendValue(DuckDBTimeOnly? value) => Append(value == null ? (DuckDBTime?)null : NativeMethods.DateTimeHelpers.DuckDBToTime(value.Value));
+    public DuckDBAppenderRow AppendValue(TimeOnly? value) => AppendValueInternal(value);
 #endif
 
-    public DuckDBAppenderRow AppendValue(DateTime? value) => Append(value == null ? (DuckDBTimestampStruct?)null : NativeMethods.DateTimeHelpers.DuckDBToTimestamp(DuckDBTimestamp.FromDateTime(value.Value)));
+    public DuckDBAppenderRow AppendValue(DuckDBDateOnly? value) => AppendValueInternal(value);
+
+    public DuckDBAppenderRow AppendValue(DuckDBTimeOnly? value) => AppendValueInternal(value);
+
+
+    public DuckDBAppenderRow AppendValue(DateTime? value) => AppendValueInternal(value);
 
     public DuckDBAppenderRow AppendValue(TimeSpan? value)
     {
-        return AppendValue2(value);
+        return AppendValueInternal(value);
     }
 
     #endregion
 
-    private DuckDBAppenderRow AppendValue2<T>(T? value)
+    private DuckDBAppenderRow AppendValueInternal<T>(T? value)
     {
         CheckColumnAccess();
 
@@ -128,24 +110,6 @@ private DuckDBAppenderRow AppendValue2<T>(T? value)
         return this;
     }
 
-    private DuckDBAppenderRow Append<T>(T? value) where T : unmanaged
-    {
-        CheckColumnAccess();
-
-        if (value == null)
-        {
-            vectorWriters[columnIndex].AppendNull(rowIndex);
-        }
-        else
-        {
-            vectorWriters[columnIndex].AppendValueInternal(value.Value, rowIndex);
-        }
-
-        columnIndex++;
-
-        return this;
-    }
-
 #if NET6_0_OR_GREATER
     private unsafe DuckDBAppenderRow AppendSpan(Span<byte> val)
     {

diff --git a/DuckDB.NET.Data/Internal/Writer/BooleanVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/BooleanVectorDataWriter.cs
@@ -0,0 +1,9 @@
+using System;
+using DuckDB.NET.Native;
+
+namespace DuckDB.NET.Data.Internal.Writer;
+
+internal sealed unsafe class BooleanVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
+{
+    internal override bool AppendBool(bool value, ulong rowIndex) => AppendValueInternal(value, rowIndex);
+}
diff --git a/DuckDB.NET.Data/Internal/Writer/DateTimeVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/DateTimeVectorDataWriter.cs
@@ -0,0 +1,19 @@
+using System;
+using DuckDB.NET.Native;
+
+namespace DuckDB.NET.Data.Internal.Writer;
+
+internal sealed unsafe class DateTimeVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
+{
+    internal override bool AppendDateTime(DateTime value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTimestamp(DuckDBTimestamp.FromDateTime(value)), rowIndex);
+
+#if NET6_0_OR_GREATER
+    internal override bool AppendDateOnly(DateOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToDate(value), rowIndex);
+
+    internal override bool AppendTimeOnly(TimeOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTime(value), rowIndex);
+#endif
+
+    internal override bool AppendDateOnly(DuckDBDateOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToDate(value), rowIndex);
+
+    internal override bool AppendTimeOnly(DuckDBTimeOnly value, ulong rowIndex) => AppendValueInternal(NativeMethods.DateTimeHelpers.DuckDBToTime(value), rowIndex);
+}
diff --git a/DuckDB.NET.Data/Internal/Writer/DecimalVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/DecimalVectorDataWriter.cs
@@ -9,7 +9,7 @@ internal sealed unsafe class DecimalVectorDataWriter(IntPtr vector, void* vector
     private readonly byte scale = NativeMethods.LogicalType.DuckDBDecimalScale(logicalType);
     private readonly DuckDBType decimalType = NativeMethods.LogicalType.DuckDBDecimalInternalType(logicalType);
 
-    public void AppendValue(decimal value, ulong rowIndex)
+    internal override bool AppendDecimal(decimal value, ulong rowIndex)
     {
         var power = Math.Pow(10, scale);
 
@@ -30,10 +30,12 @@ public void AppendValue(decimal value, ulong rowIndex)
 
                 var result = BigInteger.Multiply(new BigInteger(integralPart), new BigInteger(power));
 
-                result  += new BigInteger(decimal.Multiply(fractionalPart, (decimal)power));
-                
+                result += new BigInteger(decimal.Multiply(fractionalPart, (decimal)power));
+
                 AppendValueInternal(new DuckDBHugeInt(result), rowIndex);
                 break;
         }
+
+        return true;
     }
-}
+}
diff --git a/DuckDB.NET.Data/Internal/Writer/NumericVectorDataWriter.cs b/DuckDB.NET.Data/Internal/Writer/NumericVectorDataWriter.cs
@@ -0,0 +1,12 @@
+using System;
+using System.Numerics;
+using DuckDB.NET.Native;
+
+namespace DuckDB.NET.Data.Internal.Writer;
+
+internal sealed unsafe class NumericVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType)
+{
+    internal override bool AppendNumeric<T>(T value, ulong rowIndex) => AppendValueInternal(value, rowIndex);
+
+    internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => AppendValueInternal<DuckDBHugeInt>(new DuckDBHugeInt(value), rowIndex);
+}
diff --git a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterBase.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Numerics;
 using DuckDB.NET.Native;
 
 namespace DuckDB.NET.Data.Internal.Writer;
@@ -29,24 +30,70 @@ public unsafe void AppendValue<T>(T value, ulong rowIndex)
 
         _ = value switch
         {
+            bool val => AppendBool(val, rowIndex),
+
+            sbyte val => AppendNumeric(val, rowIndex),
+            short val => AppendNumeric(val, rowIndex),
+            int val => AppendNumeric(val, rowIndex),
+            long val => AppendNumeric(val, rowIndex),
+            byte val => AppendNumeric(val, rowIndex),
+            ushort val => AppendNumeric(val, rowIndex),
+            uint val => AppendNumeric(val, rowIndex),
+            ulong val => AppendNumeric(val, rowIndex),
+            float val => AppendNumeric(val, rowIndex),
+            double val => AppendNumeric(val, rowIndex),
+
+            decimal val => AppendDecimal(val, rowIndex),
+            BigInteger val => AppendBigInteger(val, rowIndex),
+
             string val => AppendString(val, rowIndex),
             Guid val => AppendGuid(val, rowIndex),
+            DateTime val => AppendDateTime(val, rowIndex),
             TimeSpan val => AppendTimeSpan(val, rowIndex),
-            _ => throw new InvalidOperationException($"Cannot write ${typeof(T).Name} to {columnType} column")
+            DuckDBDateOnly val => AppendDateOnly(val, rowIndex),
+            DuckDBTimeOnly val => AppendTimeOnly(val, rowIndex),
+#if NET6_0_OR_GREATER
+            DateOnly val => AppendDateOnly(val, rowIndex),
+            TimeOnly val => AppendTimeOnly(val, rowIndex),
+#endif
+            _ => ThrowException<T>()
         };
     }
 
-    internal virtual bool AppendTimeSpan(TimeSpan value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write timespan to {columnType} column");
+    internal virtual bool AppendBool(bool value, ulong rowIndex) => ThrowException<bool>();
+
+    internal virtual bool AppendDecimal(decimal value, ulong rowIndex) => ThrowException<decimal>();
+
+    internal virtual bool AppendTimeSpan(TimeSpan value, ulong rowIndex) => ThrowException<TimeSpan>();
+
+    internal virtual bool AppendGuid(Guid value, ulong rowIndex) => ThrowException<Guid>();
+
+    internal virtual bool AppendBlob(byte* value, int length, ulong rowIndex) => ThrowException<byte[]>();
+
+    internal virtual bool AppendString(string value, ulong rowIndex) => ThrowException<string>();
 
-    internal virtual bool AppendGuid(Guid value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write guid to {columnType} column");
+    internal virtual bool AppendDateTime(DateTime value, ulong rowIndex) => ThrowException<DateTime>();
 
-    internal virtual bool AppendBlob(byte* value, int length, ulong rowIndex) => throw new InvalidOperationException($"Cannot write blob to {columnType} column");
+#if NET6_0_OR_GREATER
+    internal virtual bool AppendDateOnly(DateOnly value, ulong rowIndex) => ThrowException<DateOnly>();
 
-    internal virtual bool AppendString(string value, ulong rowIndex) => throw new InvalidOperationException($"Cannot write string to {columnType} column");
+    internal virtual bool AppendTimeOnly(TimeOnly value, ulong rowIndex) => ThrowException<TimeOnly>();
+#endif
 
-    internal virtual bool AppendNumeric<T>(T value, ulong rowIndex) where T : unmanaged => throw new InvalidOperationException($"Cannot write {typeof(T).Name} to {columnType} column");
+    internal virtual bool AppendDateOnly(DuckDBDateOnly value, ulong rowIndex) => ThrowException<DuckDBDateOnly>();
+
+    internal virtual bool AppendTimeOnly(DuckDBTimeOnly value, ulong rowIndex) => ThrowException<DuckDBTimeOnly>();
+
+    internal virtual bool AppendNumeric<T>(T value, ulong rowIndex) where T : unmanaged => ThrowException<T>();
+
+    internal virtual bool AppendBigInteger(BigInteger value, ulong rowIndex) => ThrowException<BigInteger>();
+
+    private bool ThrowException<T>()
+    {
+        throw new InvalidOperationException($"Cannot write {typeof(T).Name} to {columnType} column");
+    }
 
-    public unsafe bool AppendValueInternal<T>(T value, ulong rowIndex) where T : unmanaged
+    internal unsafe bool AppendValueInternal<T>(T value, ulong rowIndex) where T : unmanaged
     {
         ((T*)vectorData)[rowIndex] = value;
         return true;

diff --git a/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs b/DuckDB.NET.Data/Internal/Writer/VectorDataWriterFactory.cs
@@ -1,21 +1,31 @@
 using System;
+using DuckDB.NET.Data.Internal.Reader;
 using DuckDB.NET.Native;
 
 namespace DuckDB.NET.Data.Internal.Writer
 {
     internal static class VectorDataWriterFactory
     {
-        public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, void* dataPointer, DuckDBLogicalType logicalType)
+        public static unsafe VectorDataWriterBase CreateWriter(IntPtr vector, DuckDBLogicalType logicalType)
         {
+            var dataPointer = NativeMethods.Vectors.DuckDBVectorGetData(vector);
             var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(logicalType);
+
             return columnType switch
             {
                 DuckDBType.Uuid => new GuidVectorDataWriter(vector, dataPointer, columnType),
+                DuckDBType.Date => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
+                DuckDBType.Time => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
+                DuckDBType.Interval => new IntervalVectorDataWriter(vector, dataPointer, columnType),
+                DuckDBType.Timestamp => new DateTimeVectorDataWriter(vector, dataPointer, columnType),
+
+                DuckDBType.Boolean => new BooleanVectorDataWriter(vector, dataPointer, columnType),
+
                 DuckDBType.Blob => new StringVectorDataWriter(vector, dataPointer, columnType),
                 DuckDBType.Varchar => new StringVectorDataWriter(vector, dataPointer, columnType),
-                DuckDBType.Interval => new IntervalVectorDataWriter(vector, dataPointer, columnType),
+
                 DuckDBType.Decimal => new DecimalVectorDataWriter(vector, dataPointer, logicalType, columnType),
-                _ => new VectorDataWriterBase(vector, dataPointer, columnType)
+                _ => new NumericVectorDataWriter(vector, dataPointer, columnType)
             };
         }
     }

diff --git a/DuckDB.NET.Test/ManagedAppenderTests.cs b/DuckDB.NET.Test/ManagedAppenderTests.cs
@@ -44,7 +44,7 @@ public void CommonTypes()
                     .AppendValue(date.AddDays(i))
                     .AppendNullValue()
                     .AppendValue(new BigInteger(ulong.MaxValue) + i)
-                    .AppendValue(new BigInteger(ulong.MaxValue) * 2 + i, true)
+                    .AppendValue(new BigInteger(ulong.MaxValue) * 2 + i)
                     .AppendValue(i + i / 100m)
                     .EndRow();
             }