Skip to content

Commit

Permalink
IGNITE-17992 .NET: Fix colocation hash for temporal types with custom…
Browse files Browse the repository at this point in the history 
… precision (#1598)

When temporal type (Time, DateTime, Timestamp) precision is less than `MAX_TIME_PRECISION`, column value is adjusted (truncated) by the server on insert, resulting in a different colocation hash. For example, nanosecond component is dropped when precision is 0.

* Propagate `Column.Precision` to .NET client (already sent by the server).
* Pass precision to `BinaryTupleBuilder` methods, and apply the same `NormalizeNanos` logic that `RowAssembler` uses.
  • Loading branch information
@ptupitsyn
ptupitsyn committed Jan 30, 2023
1 parent 5fb4ae6 commit 08171cc
Show file tree
Hide file tree
Showing 16 changed files with 358 additions and 125 deletions.
6 changes: 3 additions & 3 deletions ...ms/dotnet/Apache.Ignite.Benchmarks/Table/Serialization/SerializerHandlerBenchmarksBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ public abstract class SerializerHandlerBenchmarksBase

internal static readonly Schema Schema = new(1, 1, new[]
{
new Column(nameof(Car.Id), ClientDataType.Uuid, IsNullable: false, IsColocation: true, IsKey: true, SchemaIndex: 0, Scale: 0),
new Column(nameof(Car.BodyType), ClientDataType.String, IsNullable: false, IsColocation: false, IsKey: false, SchemaIndex: 1, Scale: 0),
new Column(nameof(Car.Seats), ClientDataType.Int32, IsNullable: false, IsColocation: false, IsKey: false, SchemaIndex: 2, Scale: 0)
new Column(nameof(Car.Id), ClientDataType.Uuid, IsNullable: false, IsColocation: true, IsKey: true, SchemaIndex: 0, Scale: 0, Precision: 0),
new Column(nameof(Car.BodyType), ClientDataType.String, IsNullable: false, IsColocation: false, IsKey: false, SchemaIndex: 1, Scale: 0, Precision: 0),
new Column(nameof(Car.Seats), ClientDataType.Int32, IsNullable: false, IsColocation: false, IsKey: false, SchemaIndex: 2, Scale: 0, Precision: 0)
});

internal static readonly byte[] SerializedData = GetSerializedData();
Expand Down
15 changes: 10 additions & 5 deletions modules/platforms/dotnet/Apache.Ignite.Tests/FakeServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,53 +340,58 @@ private void GetSchemas(MsgPackReader reader, Socket handler, long requestId)
if (tableId == ExistingTableId)
{
writer.WriteArrayHeader(1); // Columns.
writer.WriteArrayHeader(6); // Column props.
writer.WriteArrayHeader(7); // Column props.
writer.Write("ID");
writer.Write((int)ClientDataType.Int32);
writer.Write(true); // Key.
writer.Write(false); // Nullable.
writer.Write(true); // Colocation.
writer.Write(0); // Scale.
writer.Write(0); // Precision.
}
else if (tableId == CompositeKeyTableId)
{
writer.WriteArrayHeader(2); // Columns.

writer.WriteArrayHeader(6); // Column props.
writer.WriteArrayHeader(7); // Column props.
writer.Write("IdStr");
writer.Write((int)ClientDataType.String);
writer.Write(true); // Key.
writer.Write(false); // Nullable.
writer.Write(true); // Colocation.
writer.Write(0); // Scale.
writer.Write(0); // Precision.

writer.WriteArrayHeader(6); // Column props.
writer.WriteArrayHeader(7); // Column props.
writer.Write("IdGuid");
writer.Write((int)ClientDataType.Uuid);
writer.Write(true); // Key.
writer.Write(false); // Nullable.
writer.Write(true); // Colocation.
writer.Write(0); // Scale.
writer.Write(0); // Precision.
}
else if (tableId == CustomColocationKeyTableId)
{
writer.WriteArrayHeader(2); // Columns.

writer.WriteArrayHeader(6); // Column props.
writer.WriteArrayHeader(7); // Column props.
writer.Write("IdStr");
writer.Write((int)ClientDataType.String);
writer.Write(true); // Key.
writer.Write(false); // Nullable.
writer.Write(true); // Colocation.
writer.Write(0); // Scale.
writer.Write(0); // Precision.

writer.WriteArrayHeader(6); // Column props.
writer.WriteArrayHeader(7); // Column props.
writer.Write("IdGuid");
writer.Write((int)ClientDataType.Uuid);
writer.Write(true); // Key.
writer.Write(false); // Nullable.
writer.Write(false); // Colocation.
writer.Write(0); // Scale.
writer.Write(0); // Precision.
}

Send(handler, requestId, arrayBufferWriter);
Expand Down
51 changes: 26 additions & 25 deletions modules/platforms/dotnet/Apache.Ignite.Tests/Proto/BinaryTuple/BinaryTupleTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ namespace Apache.Ignite.Tests.Proto.BinaryTuple
using System.Numerics;
using Internal.Proto;
using Internal.Proto.BinaryTuple;
using Internal.Table;
using NodaTime;
using NUnit.Framework;

Expand Down Expand Up @@ -527,12 +528,12 @@ public void TestTime()
var reader = BuildAndRead(
(ref BinaryTupleBuilder b) =>
{
b.AppendTime(default);
b.AppendTime(val);
b.AppendTime(LocalTime.MinValue);
b.AppendTime(LocalTime.MaxValue);
b.AppendTime(LocalTime.Midnight);
b.AppendTime(LocalTime.Noon);
b.AppendTime(default, 0);
b.AppendTime(val, TemporalTypes.MaxTimePrecision);
b.AppendTime(LocalTime.MinValue, TemporalTypes.MaxTimePrecision);
b.AppendTime(LocalTime.MaxValue, TemporalTypes.MaxTimePrecision);
b.AppendTime(LocalTime.Midnight, 0);
b.AppendTime(LocalTime.Noon, 0);
},
6);

Expand All @@ -552,10 +553,10 @@ public void TestDateTime()
var reader = BuildAndRead(
(ref BinaryTupleBuilder b) =>
{
b.AppendDateTime(default);
b.AppendDateTime(val);
b.AppendDateTime(LocalDateTime.MaxIsoValue);
b.AppendDateTime(LocalDateTime.MinIsoValue);
b.AppendDateTime(default, 0);
b.AppendDateTime(val, TemporalTypes.MaxTimePrecision);
b.AppendDateTime(LocalDateTime.MaxIsoValue, TemporalTypes.MaxTimePrecision);
b.AppendDateTime(LocalDateTime.MinIsoValue, TemporalTypes.MaxTimePrecision);
},
4);

Expand All @@ -573,12 +574,12 @@ public void TestTimestamp()
var reader = BuildAndRead(
(ref BinaryTupleBuilder b) =>
{
b.AppendTimestamp(default);
b.AppendTimestamp(val);
b.AppendTimestamp(Instant.MaxValue);
b.AppendTimestamp(Instant.MinValue);
b.AppendTimestamp(NodaConstants.BclEpoch);
b.AppendTimestamp(NodaConstants.JulianEpoch);
b.AppendTimestamp(default, 0);
b.AppendTimestamp(val, TemporalTypes.MaxTimePrecision);
b.AppendTimestamp(Instant.MaxValue, TemporalTypes.MaxTimePrecision);
b.AppendTimestamp(Instant.MinValue, TemporalTypes.MaxTimePrecision);
b.AppendTimestamp(NodaConstants.BclEpoch, TemporalTypes.MaxTimePrecision);
b.AppendTimestamp(NodaConstants.JulianEpoch, TemporalTypes.MaxTimePrecision);
},
6);

Expand Down Expand Up @@ -738,12 +739,12 @@ public void TestAppendNullable()
b.AppendNumberNullable(null);
b.AppendDateNullable(date);
b.AppendDateNullable(null);
b.AppendTimeNullable(dateTime.TimeOfDay);
b.AppendTimeNullable(null);
b.AppendDateTimeNullable(dateTime);
b.AppendDateTimeNullable(null);
b.AppendTimestampNullable(Instant.FromDateTimeUtc(utcNow));
b.AppendTimestampNullable(null);
b.AppendTimeNullable(dateTime.TimeOfDay, TemporalTypes.MaxTimePrecision);
b.AppendTimeNullable(null, 0);
b.AppendDateTimeNullable(dateTime, TemporalTypes.MaxTimePrecision);
b.AppendDateTimeNullable(null, 0);
b.AppendTimestampNullable(Instant.FromDateTimeUtc(utcNow), TemporalTypes.MaxTimePrecision);
b.AppendTimestampNullable(null, 0);
b.AppendDurationNullable(Duration.FromMinutes(1));
b.AppendDurationNullable(null);
b.AppendPeriodNullable(Period.FromDays(1));
Expand Down Expand Up @@ -815,10 +816,10 @@ public void TestObject()
b.AppendObject(bitArray, ClientDataType.BitMask);
b.AppendObject(guid, ClientDataType.Uuid);
b.AppendObject(bytes, ClientDataType.Bytes);
b.AppendObject(LocalTime.FromMinutesSinceMidnight(123), ClientDataType.Time);
b.AppendObject(LocalTime.FromMinutesSinceMidnight(123), ClientDataType.Time, precision: TemporalTypes.MaxTimePrecision);
b.AppendObject(date, ClientDataType.Date);
b.AppendObject(dateTime, ClientDataType.DateTime);
b.AppendObject(Instant.FromDateTimeUtc(utcNow), ClientDataType.Timestamp);
b.AppendObject(dateTime, ClientDataType.DateTime, precision: TemporalTypes.MaxTimePrecision);
b.AppendObject(Instant.FromDateTimeUtc(utcNow), ClientDataType.Timestamp, precision: TemporalTypes.MaxTimePrecision);
},
17);

Expand Down
170 changes: 155 additions & 15 deletions modules/platforms/dotnet/Apache.Ignite.Tests/Proto/ColocationHashTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,18 @@ namespace Apache.Ignite.Tests.Proto;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Numerics;
using System.Reflection;
using System.Threading.Tasks;
using Ignite.Table;
using Internal.Buffers;
using Internal.Proto;
using Internal.Proto.BinaryTuple;
using Internal.Proto.MsgPack;
using Internal.Table;
using Internal.Table.Serialization;
using NodaTime;
using NUnit.Framework;

Expand Down Expand Up @@ -86,7 +94,8 @@ public class ColocationHashTests : IgniteTestsBase
new LocalDate(2, 1, 1),
new LocalDate(1, 1, 1),
default(LocalDate),
new LocalTime(9, 8, 7),
new LocalTime(9, 8, 7, 6),
LocalTime.FromHourMinuteSecondNanosecond(hour: 1, minute: 2, second: 3, nanosecondWithinSecond: 456789),
LocalTime.Midnight,
LocalTime.Noon,
LocalDateTime.FromDateTime(DateTime.UtcNow).TimeOfDay,
Expand All @@ -102,48 +111,179 @@ public class ColocationHashTests : IgniteTestsBase
[Test]
[TestCaseSource(nameof(TestCases))]
public async Task TestSingleKeyColocationHashIsSameOnServerAndClient(object key) =>
await AssertClientAndServerHashesAreEqual(key);
await AssertClientAndServerHashesAreEqual(keys: key);

[Test]
public async Task TestSingleKeyColocationHashIsSameOnServerAndClientCustomTimePrecision(
[Values(0, 1, 3, 4, 5, 6, 7, 8, 9)] int timePrecision,
[Values(0, 1, 3, 6)] int timestampPrecision)
{
foreach (var t in TestCases)
{
await AssertClientAndServerHashesAreEqual(timePrecision, timestampPrecision, t);
}
}

[Test]
public async Task TestLocalTimeColocationHashIsSameOnServerAndClient([Values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)] int timePrecision) =>
await AssertClientAndServerHashesAreEqual(timePrecision, keys: LocalTime.FromHourMinuteSecondNanosecond(11, 33, 44, 123_456));

[Test]
public async Task TestLocalDateTimeColocationHashIsSameOnServerAndClient([Values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)] int timePrecision) =>
await AssertClientAndServerHashesAreEqual(timePrecision, keys: new LocalDateTime(2022, 01, 27, 1, 2, 3, 999));

[Test]
public async Task TestTimestampColocationHashIsSameOnServerAndClient(
[Values(0, 1, 2, 3, 4, 5, 6)] int timestampPrecision) =>
await AssertClientAndServerHashesAreEqual(timestampPrecision: timestampPrecision, keys: Instant.FromDateTimeUtc(DateTime.UtcNow));

[Test]
public async Task TestMultiKeyColocationHashIsSameOnServerAndClient()
{
for (var i = 0; i < TestCases.Length; i++)
{
await AssertClientAndServerHashesAreEqual(TestCases.Take(i + 1).ToArray());
await AssertClientAndServerHashesAreEqual(TestCases.Skip(i).ToArray());
await AssertClientAndServerHashesAreEqual(keys: TestCases.Take(i + 1).ToArray());
await AssertClientAndServerHashesAreEqual(keys: TestCases.Skip(i).ToArray());
}
}

[Test]
public async Task TestMultiKeyColocationHashIsSameOnServerAndClientCustomTimePrecision(
[Values(0, 1, 4, 5, 6, 7, 8, 9)] int timePrecision,
[Values(0, 1, 3, 6)] int timestampPrecision)
{
for (var i = 0; i < TestCases.Length; i++)
{
await AssertClientAndServerHashesAreEqual(timePrecision, timestampPrecision, TestCases.Take(i + 1).ToArray());
await AssertClientAndServerHashesAreEqual(timePrecision, timestampPrecision, TestCases.Skip(i).ToArray());
}
}

private static (byte[] Bytes, int Hash) WriteAsBinaryTuple(IReadOnlyCollection<object> arr)
private static (byte[] Bytes, int Hash) WriteAsBinaryTuple(IReadOnlyCollection<object> arr, int timePrecision, int timestampPrecision)
{
using var builder = new BinaryTupleBuilder(arr.Count * 3, hashedColumnsPredicate: new TestIndexProvider());
using var builder = new BinaryTupleBuilder(arr.Count * 3, hashedColumnsPredicate: new TestIndexProvider(x => x % 3 == 2));

foreach (var obj in arr)
{
builder.AppendObjectWithType(obj);
builder.AppendObjectWithType(obj, timePrecision, timestampPrecision);
}

return (builder.Build().ToArray(), builder.Hash);
}

private async Task AssertClientAndServerHashesAreEqual(params object[] keys)
private static int WriteAsIgniteTuple(IReadOnlyCollection<object> arr, int timePrecision, int timestampPrecision)
{
var (bytes, hash) = WriteAsBinaryTuple(keys);
var igniteTuple = new IgniteTuple();
int i = 1;

foreach (var obj in arr)
{
igniteTuple["m_Item" + i++] = obj;
}

var serverHash = await GetServerHash(bytes, keys.Length);
var builder = new BinaryTupleBuilder(arr.Count, hashedColumnsPredicate: new TestIndexProvider(_ => true));

try
{
var schema = GetSchema(arr, timePrecision, timestampPrecision);
var noValueSet = new byte[arr.Count].AsSpan();

Assert.AreEqual(serverHash, hash, string.Join(", ", keys));
TupleSerializerHandler.Instance.Write(ref builder, igniteTuple, schema, arr.Count, noValueSet);
return builder.Hash;
}
finally
{
builder.Dispose();
}
}

[SuppressMessage("ReSharper", "UnusedMember.Local", Justification = "Used by reflection.")]
private static int WriteAsPoco<T>(T obj, int timePrecision, int timestampPrecision)
{
var poco = Tuple.Create(obj);
IRecordSerializerHandler<Tuple<T>> handler = new ObjectSerializerHandler<Tuple<T>>();
var schema = GetSchema(new object[] { obj! }, timePrecision, timestampPrecision);

using var buf = new PooledArrayBuffer();
var writer = new MsgPackWriter(buf);

return handler.Write(ref writer, schema, poco, computeHash: true);
}

private static Schema GetSchema(IReadOnlyCollection<object> arr, int timePrecision, int timestampPrecision)
{
var columns = arr.Select((obj, ci) => GetColumn(obj, ci, timePrecision, timestampPrecision)).ToArray();

return new Schema(Version: 0, arr.Count, columns);
}

private static Column GetColumn(object value, int schemaIndex, int timePrecision, int timestampPrecision)
{
var colType = value switch
{
sbyte => ClientDataType.Int8,
short => ClientDataType.Int16,
int => ClientDataType.Int32,
long => ClientDataType.Int64,
float => ClientDataType.Float,
double => ClientDataType.Double,
decimal => ClientDataType.Decimal,
Guid => ClientDataType.Uuid,
byte[] => ClientDataType.Bytes,
string => ClientDataType.String,
BigInteger => ClientDataType.Number,
BitArray => ClientDataType.BitMask,
LocalTime => ClientDataType.Time,
LocalDate => ClientDataType.Date,
LocalDateTime => ClientDataType.DateTime,
Instant => ClientDataType.Timestamp,
_ => throw new Exception("Unknown type: " + value.GetType())
};

var precision = colType switch
{
ClientDataType.Time => timePrecision,
ClientDataType.DateTime => timePrecision,
ClientDataType.Timestamp => timestampPrecision,
_ => 0
};

var scale = value is decimal d ? BitConverter.GetBytes(decimal.GetBits(d)[3])[2] : 0;

return new Column("m_Item" + (schemaIndex + 1), colType, false, true, true, schemaIndex, Scale: scale, precision);
}

private async Task AssertClientAndServerHashesAreEqual(int timePrecision = 9, int timestampPrecision = 6, params object[] keys)
{
var (bytes, clientHash) = WriteAsBinaryTuple(keys, timePrecision, timestampPrecision);
var clientHash2 = WriteAsIgniteTuple(keys, timePrecision, timestampPrecision);

var serverHash = await GetServerHash(bytes, keys.Length, timePrecision, timestampPrecision);

var msg = $"Time precision: {timePrecision}, timestamp precision: {timestampPrecision}, keys: {string.Join(", ", keys)}";

Assert.AreEqual(serverHash, clientHash, $"Server hash mismatch. {msg}");
Assert.AreEqual(clientHash, clientHash2, $"IgniteTuple hash mismatch. {msg}");

if (keys.Length == 1)
{
var obj = keys[0];
var method = GetType().GetMethod("WriteAsPoco", BindingFlags.Static | BindingFlags.NonPublic)!.MakeGenericMethod(obj.GetType());
var clientHash3 = (int)method.Invoke(null, new[] { obj, timePrecision, timestampPrecision })!;

Assert.AreEqual(clientHash, clientHash3, $"Poco hash mismatch. {msg}");
}
}

private async Task<int> GetServerHash(byte[] bytes, int count)
private async Task<int> GetServerHash(byte[] bytes, int count, int timePrecision, int timestampPrecision)
{
var nodes = await Client.GetClusterNodesAsync();

return await Client.Compute.ExecuteAsync<int>(nodes, ColocationHashJob, count, bytes);
return await Client.Compute.ExecuteAsync<int>(nodes, ColocationHashJob, count, bytes, timePrecision, timestampPrecision);
}

private class TestIndexProvider : IHashedColumnIndexProvider
private record TestIndexProvider(Func<int, bool> Delegate) : IHashedColumnIndexProvider
{
public bool IsHashedColumnIndex(int index) => index % 3 == 2;
public bool IsHashedColumnIndex(int index) => Delegate(index);
}
}

0 comments on commit 08171cc

Please sign in to comment.