From 413e4c70d3a82d615045a0552a967a45878defc7 Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Fri, 6 Jun 2025 16:55:44 +0200 Subject: [PATCH 1/3] Array Properties: Serialization --- src/Weaviate.Client.Tests/Helpers.cs | 13 +- .../Integration/Properties.cs | 87 ++++++ src/Weaviate.Client.Tests/Unit/Filters.cs | 4 +- src/Weaviate.Client.Tests/Unit/Properties.cs | 41 +++ src/Weaviate.Client.Tests/Unit/_Unit.cs | 29 +- .../{Helpers.cs => ConnectionHelpers.cs} | 0 src/Weaviate.Client/DataClient.cs | 252 +++++++++++++++- src/Weaviate.Client/Extensions.cs | 123 ++++---- src/Weaviate.Client/Models/DataTypes.cs | 41 +++ src/Weaviate.Client/Models/Defaults.cs | 7 - src/Weaviate.Client/Models/Filter.cs | 6 +- .../Models/FilterExpression.cs | 2 +- src/Weaviate.Client/Models/Property.cs | 280 +++++++++++++----- src/Weaviate.Client/Models/WeaviateObject.cs | 66 +---- src/Weaviate.Client/gRPC/Client.cs | 19 +- 15 files changed, 735 insertions(+), 235 deletions(-) create mode 100644 src/Weaviate.Client.Tests/Integration/Properties.cs create mode 100644 src/Weaviate.Client.Tests/Unit/Properties.cs rename src/Weaviate.Client/{Helpers.cs => ConnectionHelpers.cs} (100%) create mode 100644 src/Weaviate.Client/Models/DataTypes.cs delete mode 100644 src/Weaviate.Client/Models/Defaults.cs diff --git a/src/Weaviate.Client.Tests/Helpers.cs b/src/Weaviate.Client.Tests/Helpers.cs index 179597b5..ff6462c1 100644 --- a/src/Weaviate.Client.Tests/Helpers.cs +++ b/src/Weaviate.Client.Tests/Helpers.cs @@ -9,7 +9,10 @@ public LoggingHandler(Action log) _log = log; } - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) { _log($"Request: {request.Method} {request.RequestUri}"); @@ -19,7 +22,11 @@ protected override async Task SendAsync(HttpRequestMessage _log($"Request Content: {requestContent}"); // Buffer the content so it can be read again. - request.Content = new StringContent(requestContent, System.Text.Encoding.UTF8, "application/json"); + request.Content = new StringContent( + requestContent, + System.Text.Encoding.UTF8, + "application/json" + ); } foreach (var header in request.Headers) @@ -44,4 +51,4 @@ protected override async Task SendAsync(HttpRequestMessage return response; } -} \ No newline at end of file +} diff --git a/src/Weaviate.Client.Tests/Integration/Properties.cs b/src/Weaviate.Client.Tests/Integration/Properties.cs new file mode 100644 index 00000000..e14c171e --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/Properties.cs @@ -0,0 +1,87 @@ +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + public record TestProperties + { + public string? TestText { get; set; } + public string[]? TestTextArray { get; set; } + public int? TestInt { get; set; } + public int[]? TestIntArray { get; set; } + public bool? TestBool { get; set; } + public bool[]? TestBoolArray { get; set; } + public double? TestNumber { get; set; } + public double[]? TestNumberArray { get; set; } + public DateTime? TestDate { get; set; } + public DateTime[]? TestDateArray { get; set; } + public Guid? TestUuid { get; set; } + public Guid[]? TestUuidArray { get; set; } + public GeoCoordinate? TestGeo { get; set; } + // // public byte[]? TestBlob { get; set; } + // // public PhoneNumber? TestPhone { get; set; } + // // public object? TestObject { get; set; } + // // public object? TestObjectArray { get; set; } + } + + [Fact] + public async Task AllPropertiesSaveRetrieve() + { + Property[] props = + [ + Property.Text("testText"), + Property.TextArray("testTextArray"), + Property.Int("testInt"), + Property.IntArray("testIntArray"), + Property.Bool("testBool"), + Property.BoolArray("testBoolArray"), + Property.Number("testNumber"), + Property.NumberArray("testNumberArray"), + Property.Date("testDate"), + Property.DateArray("testDateArray"), + Property.Uuid("testUuid"), + Property.UuidArray("testUuidArray"), + Property.GeoCoordinate("testGeo"), + // Property.Blob("testBlob"), + // Property.PhoneNumber("testPhone"), + // Property.Object("testObject"), + // Property.ObjectArray("testObjectArray"), + ]; + + // 1. Create collection + var c = await CollectionFactory( + description: "Testing collection properties", + properties: props + ); + + // 2. Create an object with values for all properties + var testData = new TestProperties + { + TestText = "dummyText", + TestTextArray = new[] { "dummyTextArray1", "dummyTextArray2" }, + TestInt = 123, + TestIntArray = new[] { 1, 2, 3 }, + TestBool = true, + TestBoolArray = new[] { true, false }, + TestNumber = 456.789, + TestNumberArray = new[] { 4.5, 6.7 }, + TestDate = DateTime.Now.AddDays(-1), + TestDateArray = new[] { DateTime.Now.AddDays(-2), DateTime.Now.AddDays(-3) }, + TestUuid = Guid.NewGuid(), + TestUuidArray = new[] { Guid.NewGuid(), Guid.NewGuid() }, + TestGeo = new GeoCoordinate(12.345f, 67.890f), + }; + + var id = await c.Data.Insert(testData); + + // 3. Retrieve the object and confirm all properties match + var retrieved = await c.Query.FetchObjectByID(id); + + var obj = retrieved.Objects.First(); + + var concreteObj = obj.As(); + + Assert.Equivalent(testData, concreteObj); + } +} diff --git a/src/Weaviate.Client.Tests/Unit/Filters.cs b/src/Weaviate.Client.Tests/Unit/Filters.cs index 750504f8..05113efc 100644 --- a/src/Weaviate.Client.Tests/Unit/Filters.cs +++ b/src/Weaviate.Client.Tests/Unit/Filters.cs @@ -12,7 +12,7 @@ public partial class UnitTests new[] { "Equal", "NotEqual", "ContainsAny" }, new[] { "GreaterThan", "GreaterThanEqual", "LessThan", "LessThanEqual" } )] - public async Task TypeSupportedOperations( + public void TypeSupportedOperations( Type t, string[] expectedMethodList, string[] unexpectedMethodList @@ -32,8 +32,6 @@ string[] unexpectedMethodList // Assert Assert.Subset(actualMethods, methods); Assert.Empty(actualMethods.Intersect(unexpectedMethodList)); - - await Task.Yield(); } [Fact] diff --git a/src/Weaviate.Client.Tests/Unit/Properties.cs b/src/Weaviate.Client.Tests/Unit/Properties.cs new file mode 100644 index 00000000..79e57c1a --- /dev/null +++ b/src/Weaviate.Client.Tests/Unit/Properties.cs @@ -0,0 +1,41 @@ +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests; + +public partial class UnitTests +{ + public static IEnumerable CaseKeys => Cases.Keys.Select(k => new object[] { k }); + + private static Dictionary, Func)> Cases = + new() + { + ["Text"] = (Property.Text, Property.For), + ["TextArray"] = (Property.TextArray, Property.For), + ["Int"] = (Property.Int, Property.For), + ["IntArray"] = (Property.IntArray, Property.For), + ["Bool"] = (Property.Bool, Property.For), + ["BoolArray"] = (Property.BoolArray, Property.For), + ["Number"] = (Property.Number, Property.For), + ["NumberArray"] = (Property.NumberArray, Property.For), + ["Date"] = (Property.Date, Property.For), + ["DateArray"] = (Property.DateArray, Property.For), + ["Uuid"] = (Property.Uuid, Property.For), + ["UuidArray"] = (Property.UuidArray, Property.For), + ["Geo"] = (Property.GeoCoordinate, Property.For), + ["Phone"] = (Property.PhoneNumber, Property.For), + // TODO Add support for the properties below + // ["Blob"] = (Property.Blob, Property.For), + // ["Object"] = (Property.Object, Property.For), + // ["ObjectArray"] = (Property.ObjectArray, Property.For), + }; + + [Theory] + [MemberData(nameof(CaseKeys))] + public void Properties(string test) + { + var (f1, f2) = Cases[test]; + var (p1, p2) = (f1(test), f2(test)); + + Assert.Equivalent(p1, p2); + } +} diff --git a/src/Weaviate.Client.Tests/Unit/_Unit.cs b/src/Weaviate.Client.Tests/Unit/_Unit.cs index 4d61592f..00081ae8 100644 --- a/src/Weaviate.Client.Tests/Unit/_Unit.cs +++ b/src/Weaviate.Client.Tests/Unit/_Unit.cs @@ -1,3 +1,5 @@ +using System.Dynamic; +using System.Text.Json; using Weaviate.Client.Models; namespace Weaviate.Client.Tests; @@ -33,6 +35,19 @@ public void MetadataQueryImplicitConversion() Assert.Equal(q3.Vectors, vectors); } + [Fact] + public void TestBuildDynamicObjectPropertiesGeoCoordinate() + { + var geo = new { TestingPropertyType = new GeoCoordinate(12.345f, 67.890f) }; + var props = ObjectHelper.BuildDataTransferObject(geo); + + dynamic? concrete = ObjectHelper.UnmarshallProperties(props); + + Assert.NotNull(concrete); + Assert.Equal(geo.TestingPropertyType.Latitude, concrete!.TestingPropertyType.Latitude); + Assert.Equal(geo.TestingPropertyType.Longitude, concrete!.TestingPropertyType.Longitude); + } + [Fact] public void TestBuildDynamicObject() { @@ -58,7 +73,7 @@ public void TestBuildDynamicObject() }; // Act - var obj = review.Select(r => DataClient.BuildDynamicObject(r)).ToList(); + var obj = review.Select(r => ObjectHelper.BuildDataTransferObject(r)).ToList(); // Assert Assert.Equal("kineticandroid", obj[0]["author_username"]); @@ -76,6 +91,7 @@ public void TestBuildDynamicObject() [Theory] [InlineData(typeof(bool), true)] + [InlineData(typeof(bool[]), true)] [InlineData(typeof(char), true)] [InlineData(typeof(sbyte), true)] [InlineData(typeof(byte), true)] @@ -89,9 +105,20 @@ public void TestBuildDynamicObject() [InlineData(typeof(double), true)] [InlineData(typeof(double?), true)] [InlineData(typeof(decimal), true)] + [InlineData(typeof(short[]), true)] + [InlineData(typeof(ushort[]), true)] + [InlineData(typeof(int[]), true)] + [InlineData(typeof(uint[]), true)] + [InlineData(typeof(long[]), true)] + [InlineData(typeof(ulong[]), true)] + [InlineData(typeof(float[]), true)] + [InlineData(typeof(double[]), true)] + [InlineData(typeof(decimal[]), true)] [InlineData(typeof(string), true)] + [InlineData(typeof(string[]), true)] [InlineData(typeof(DateTime), true)] [InlineData(typeof(Object), false)] + [InlineData(typeof(Object[]), false)] [InlineData(typeof(WeaviateObject), false)] public void TestIsNativeTypeCheck(Type type, bool expected) { diff --git a/src/Weaviate.Client/Helpers.cs b/src/Weaviate.Client/ConnectionHelpers.cs similarity index 100% rename from src/Weaviate.Client/Helpers.cs rename to src/Weaviate.Client/ConnectionHelpers.cs diff --git a/src/Weaviate.Client/DataClient.cs b/src/Weaviate.Client/DataClient.cs index de6beb20..b35886e3 100644 --- a/src/Weaviate.Client/DataClient.cs +++ b/src/Weaviate.Client/DataClient.cs @@ -1,5 +1,8 @@ using System.Collections.Frozen; +using System.ComponentModel; +using System.Diagnostics; using System.Dynamic; +using System.Reflection; using System.Text.Json; using Google.Protobuf.WellKnownTypes; using Weaviate.Client.Models; @@ -7,29 +10,214 @@ namespace Weaviate.Client; -public class DataClient +internal class ObjectHelper { - private readonly CollectionClient _collectionClient; - private WeaviateClient _client => _collectionClient.Client; - private string _collectionName => _collectionClient.Name; + internal static T? UnmarshallProperties(IDictionary dict) + where T : new() + { + ArgumentNullException.ThrowIfNull(dict); - internal DataClient(CollectionClient collectionClient) + // Create an instance of T using the default constructor + var instance = new T(); + + if (instance is IDictionary target) + { + foreach (var kvp in dict) + { + if (kvp.Value is IDictionary subDict) + { + object? nestedValue = UnmarshallProperties(subDict); + + target[kvp.Key.Capitalize()] = nestedValue ?? subDict; + } + else + { + if (kvp.Value?.GetType() == typeof(Rest.Dto.GeoCoordinates)) + { + var value = (Rest.Dto.GeoCoordinates)kvp.Value; + target[kvp.Key.Capitalize()] = new GeoCoordinate( + value.Latitude ?? 0f, + value.Longitude ?? 0f + ); + } + else + { + target[kvp.Key.Capitalize()] = kvp.Value; + } + } + } + return instance; + } + + var type = typeof(T); + var properties = type.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanWrite) + .ToArray(); + + foreach (var property in properties) + { + var matchingKey = dict.Keys.FirstOrDefault(k => + string.Equals(k, property.Name, StringComparison.OrdinalIgnoreCase) + ); + + if (matchingKey is null) + { + continue; + } + + var value = dict[matchingKey]; + + try + { + var convertedValue = ConvertValue(value, property.PropertyType); + property.SetValue(instance, convertedValue); + } + catch (Exception ex) + { + // Skip if conversion fails + Debug.WriteLine($"Failed to convert property {property.Name}: {ex.Message}"); + continue; + } + } + + return instance; + } + + private static object? ConvertValue(object? value, System.Type targetType) { - _collectionClient = collectionClient; + // Handle null values + if (value == null) + { + if (IsNullableType(targetType) || !targetType.IsValueType) + { + return null; + } + // For non-nullable value types, return default value + return Activator.CreateInstance(targetType); + } + + // If types already match, return as-is + if (targetType.IsAssignableFrom(value.GetType())) + { + return value; + } + + // Handle nullable types + if (IsNullableType(targetType)) + { + var underlyingType = Nullable.GetUnderlyingType(targetType)!; + return ConvertValue(value, underlyingType); + } + + // Handle nested objects (dictionaries -> custom types) + if ( + value is IDictionary nestedDict + && !typeof(IDictionary).IsAssignableFrom(targetType) + ) + { + var method = typeof(ObjectHelper) + .GetMethod("UnmarshallProperties", BindingFlags.Static | BindingFlags.NonPublic)! + .MakeGenericMethod(targetType); + return method.Invoke(null, new object[] { nestedDict }); + } + + // Handle collections + if ( + IsCollectionType(targetType) + && value is System.Collections.IEnumerable enumerable + && !(value is string) + ) + { + return ConvertCollection(enumerable, targetType); + } + + // Handle enums + if (targetType.IsEnum) + { + if (value is string stringValue) + { + return System.Enum.Parse(targetType, stringValue, true); + } + return System.Enum.ToObject(targetType, value); + } + + // Try TypeConverter first (handles more cases than Convert.ChangeType) + var converter = TypeDescriptor.GetConverter(targetType); + if (converter.CanConvertFrom(value.GetType())) + { + return converter.ConvertFrom(value); + } + + // Fallback to Convert.ChangeType for basic types + return Convert.ChangeType(value, targetType); } - public static IDictionary[] MakeBeacons(params Guid[] guids) + private static bool IsNullableType(System.Type type) { - return - [ - .. guids.Select(uuid => new Dictionary + return type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); + } + + private static bool IsCollectionType(System.Type type) + { + return type.IsArray + || ( + type.IsGenericType + && ( + type.GetGenericTypeDefinition() == typeof(List<>) + || type.GetGenericTypeDefinition() == typeof(IList<>) + || type.GetGenericTypeDefinition() == typeof(ICollection<>) + || type.GetGenericTypeDefinition() == typeof(IEnumerable<>) + ) + ); + } + + private static object? ConvertCollection( + System.Collections.IEnumerable source, + System.Type targetType + ) + { + if (targetType.IsArray) + { + var elementType = targetType.GetElementType()!; + var items = new List(); + + foreach (var item in source) { - { "beacon", $"weaviate://localhost/{uuid}" }, - }), - ]; + items.Add(ConvertValue(item, elementType)); + } + + var array = Array.CreateInstance(elementType, items.Count); + for (int i = 0; i < items.Count; i++) + { + array.SetValue(items[i], i); + } + return array; + } + + if (targetType.IsGenericType) + { + var elementType = targetType.GetGenericArguments()[0]; + var listType = typeof(List<>).MakeGenericType(elementType); + var list = (System.Collections.IList)Activator.CreateInstance(listType)!; + + foreach (var item in source) + { + list.Add(ConvertValue(item, elementType)); + } + + return list; + } + + // Fallback - convert to object array + var fallbackItems = new List(); + foreach (var item in source) + { + fallbackItems.Add(item); + } + return fallbackItems.ToArray(); } - internal static IDictionary BuildDynamicObject(object? data) + internal static IDictionary BuildDataTransferObject(object? data) { var obj = new ExpandoObject(); var propDict = obj as IDictionary; @@ -54,14 +242,46 @@ public static IDictionary[] MakeBeacons(params Guid[] guids) { propDict[propertyInfo.Name] = value; } + else if (propertyInfo.PropertyType == typeof(GeoCoordinate)) + { + var newValue = (GeoCoordinate)value; + propDict[propertyInfo.Name] = new GeoCoordinates + { + Latitude = newValue.Latitude, + Longitude = newValue.Longitude, + }; + } else { - propDict[propertyInfo.Name] = BuildDynamicObject(value); // recursive call + propDict[propertyInfo.Name] = BuildDataTransferObject(value); // recursive call } } return obj; } +} + +public class DataClient +{ + private readonly CollectionClient _collectionClient; + private WeaviateClient _client => _collectionClient.Client; + private string _collectionName => _collectionClient.Name; + + internal DataClient(CollectionClient collectionClient) + { + _collectionClient = collectionClient; + } + + public static IDictionary[] MakeBeacons(params Guid[] guids) + { + return + [ + .. guids.Select(uuid => new Dictionary + { + { "beacon", $"weaviate://localhost/{uuid}" }, + }), + ]; + } // Helper method to convert C# objects to protobuf Values public static Value ConvertToValue(object obj) @@ -128,7 +348,7 @@ public async Task Insert( string? tenant = null ) { - var propDict = BuildDynamicObject(data); + var propDict = ObjectHelper.BuildDataTransferObject(data); foreach (var kvp in references ?? []) { diff --git a/src/Weaviate.Client/Extensions.cs b/src/Weaviate.Client/Extensions.cs index 98416791..d26f3aa1 100644 --- a/src/Weaviate.Client/Extensions.cs +++ b/src/Weaviate.Client/Extensions.cs @@ -13,63 +13,6 @@ public static class WeaviateExtensions WriteIndented = true, // For readability }; - private static T? UnmarshallProperties(IDictionary dict) - { - if (dict == null) - throw new ArgumentNullException(nameof(dict)); - - // Create an instance of T using the default constructor - var props = Activator.CreateInstance(); - - if (props is IDictionary target) - { - foreach (var kvp in dict) - { - if (kvp.Value is IDictionary subDict) - { - dynamic? v = UnmarshallProperties(subDict); - - target[Capitalize(kvp.Key)] = v ?? subDict; - } - else - { - target[Capitalize(kvp.Key)] = kvp.Value; - } - } - return props; - } - - var type = typeof(T); - var properties = type.GetProperties(); - - foreach (var property in properties) - { - var matchingKey = dict.Keys.FirstOrDefault(k => - string.Equals(k, property.Name, StringComparison.OrdinalIgnoreCase) - ); - - if (matchingKey != null) - { - var value = dict[matchingKey]; - if (value != null) - { - try - { - var convertedValue = Convert.ChangeType(value, property.PropertyType); - property.SetValue(props, convertedValue); - } - catch - { - // Skip if conversion fails - continue; - } - } - } - } - - return props; - } - internal static Rest.Dto.Class ToDto(this Collection collection) { var data = new Rest.Dto.Class() @@ -279,7 +222,9 @@ internal static IEnumerable FromByteString(this Google.Protobuf.ByteString // Temporary variable to hold the read object before casting object value = Type.GetTypeCode(typeof(T)) switch { + TypeCode.Int16 => reader.ReadInt16(), TypeCode.Int32 => reader.ReadInt32(), + TypeCode.Int64 => reader.ReadInt64(), TypeCode.Single => reader.ReadSingle(), TypeCode.Double => reader.ReadDouble(), TypeCode.String => reader.ReadString(), @@ -346,12 +291,17 @@ public static string Decapitalize(this string str) public static bool IsNativeType(this Type type) { - if (type.IsValueType && !type.IsClass) + // Handle nullable types + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + + // Check basic value types (excluding structs that aren't primitives) + if (underlyingType.IsPrimitive) { return true; } - switch (Type.GetTypeCode(type)) + // Check common .NET types using TypeCode + switch (Type.GetTypeCode(underlyingType)) { case TypeCode.Boolean: case TypeCode.Char: @@ -369,11 +319,56 @@ public static bool IsNativeType(this Type type) case TypeCode.String: case TypeCode.DateTime: return true; - case TypeCode.Empty: - case TypeCode.Object: - case TypeCode.DBNull: - default: - return false; } + + // Check for other common native types + if ( + underlyingType == typeof(Guid) + || underlyingType == typeof(TimeSpan) + || underlyingType == typeof(DateTimeOffset) + || underlyingType == typeof(DateTime) + || underlyingType == typeof(Uri) + ) + { + return true; + } + + // Check for arrays of native types + if (type.IsArray) + { + return type.GetElementType()?.IsNativeType() == true; + } + + // Check for generic IEnumerable where T is native + if (type.IsGenericType) + { + var genericDefinition = type.GetGenericTypeDefinition(); + + // Handle common generic collection types + if ( + genericDefinition == typeof(IEnumerable<>) + || genericDefinition == typeof(ICollection<>) + || genericDefinition == typeof(IList<>) + || genericDefinition == typeof(List<>) + || genericDefinition == typeof(HashSet<>) + || genericDefinition == typeof(ISet<>) + || genericDefinition == typeof(Queue<>) + || genericDefinition == typeof(Stack<>) + ) + { + var elementType = type.GetGenericArguments()[0]; + return elementType.IsNativeType(); + } + } + + // Check for non-generic IEnumerable (less precise, but handles ArrayList, etc.) + if (typeof(System.Collections.IEnumerable).IsAssignableFrom(type) && type != typeof(string)) + { + // For non-generic collections, we can't determine the element type at compile time + // Consider them as native for serialization purposes. + return true; + } + + return false; } } diff --git a/src/Weaviate.Client/Models/DataTypes.cs b/src/Weaviate.Client/Models/DataTypes.cs new file mode 100644 index 00000000..c92c73fa --- /dev/null +++ b/src/Weaviate.Client/Models/DataTypes.cs @@ -0,0 +1,41 @@ +namespace Weaviate.Client.Models; + +public record GeoCoordinate(float Latitude, float Longitude); + +public partial record PhoneNumber +{ + /// + /// The raw input as the phone number is present in your raw data set. It will be parsed into the standardized formats if valid. + /// + public string? Input { get; set; } = default!; + + /// + /// Read-only. Parsed result in the international format (e.g. +49 123 ...) + /// + public string? InternationalFormatted { get; set; } = default!; + + /// + /// Optional. The ISO 3166-1 alpha-2 country code. This is used to figure out the correct countryCode and international format if only a national number (e.g. 0123 4567) is provided + /// + public string? DefaultCountry { get; set; } = default!; + + /// + /// Read-only. The numerical country code (e.g. 49) + /// + public double? CountryCode { get; set; } = default!; + + /// + /// Read-only. The numerical representation of the national part + /// + public double? National { get; set; } = default!; + + /// + /// Read-only. Parsed result in the national format (e.g. 0123 456789) + /// + public string? NationalFormatted { get; set; } = default!; + + /// + /// Read-only. Indicates whether the parsed number is a valid phone number + /// + public bool? Valid { get; set; } = default!; +} diff --git a/src/Weaviate.Client/Models/Defaults.cs b/src/Weaviate.Client/Models/Defaults.cs deleted file mode 100644 index 6f99b7c5..00000000 --- a/src/Weaviate.Client/Models/Defaults.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Weaviate.Client.Models -{ - public static class Defaults - { - public static string DefaultNamedVector = "default"; - } -} \ No newline at end of file diff --git a/src/Weaviate.Client/Models/Filter.cs b/src/Weaviate.Client/Models/Filter.cs index 033db04a..d0ceee15 100644 --- a/src/Weaviate.Client/Models/Filter.cs +++ b/src/Weaviate.Client/Models/Filter.cs @@ -2,7 +2,7 @@ namespace Weaviate.Client.Models; -public record GeoCoordinatesConstraint(float Latitude, float Longitude, float Distance); +public record GeoCoordinateConstraint(float Latitude, float Longitude, float Distance); public interface IFilterEquality { @@ -148,7 +148,7 @@ internal Filter WithValue(T value) value switch { bool v => f => f.ValueBoolean = v, - GeoCoordinatesConstraint v => f => + GeoCoordinateConstraint v => f => f.ValueGeo = new GeoCoordinatesFilter { Distance = v.Distance, @@ -234,7 +234,7 @@ public Filter LessThan(T value) => public Filter LessThanEqual(T value) => WithOperator(Filters.Types.Operator.LessThanEqual).WithValue(value); - public Filter WithinGeoRange(GeoCoordinatesConstraint value) => + public Filter WithinGeoRange(GeoCoordinateConstraint value) => WithOperator(Filters.Types.Operator.WithinGeoRange).WithValue(value); public Filter Like(T value) => WithOperator(Filters.Types.Operator.Like).WithValue(value); diff --git a/src/Weaviate.Client/Models/FilterExpression.cs b/src/Weaviate.Client/Models/FilterExpression.cs index 416090e4..d906f3a3 100644 --- a/src/Weaviate.Client/Models/FilterExpression.cs +++ b/src/Weaviate.Client/Models/FilterExpression.cs @@ -29,7 +29,7 @@ internal PropertyFilter(string name) public Filter LessThanEqual(TResult value) => _prop.LessThanEqual(value); - public Filter WithinGeoRange(GeoCoordinatesConstraint value) => _prop.WithinGeoRange(value); + public Filter WithinGeoRange(GeoCoordinateConstraint value) => _prop.WithinGeoRange(value); public Filter Like(TResult value) => _prop.Like(value); diff --git a/src/Weaviate.Client/Models/Property.cs b/src/Weaviate.Client/Models/Property.cs index 0812323e..a6663d0a 100644 --- a/src/Weaviate.Client/Models/Property.cs +++ b/src/Weaviate.Client/Models/Property.cs @@ -1,37 +1,179 @@ namespace Weaviate.Client.Models; +internal class PropertyHelper +{ + internal static Property For(Type t, string name) + { + // Handle nullable types - get the underlying type + Type actualType = Nullable.GetUnderlyingType(t) ?? t; + + // Handle special types first + if (actualType == typeof(Guid)) + { + return Property.Uuid(name); + } + + if (actualType == typeof(GeoCoordinate)) + { + return Property.GeoCoordinate(name); + } + + if (actualType == typeof(PhoneNumber)) + { + return Property.PhoneNumber(name); + } + + // Handle primitive types + Func? f = Type.GetTypeCode(actualType) switch + { + TypeCode.String => Property.Text, + TypeCode.Int16 => Property.Int, + TypeCode.UInt16 => Property.Int, + TypeCode.Int32 => Property.Int, + TypeCode.UInt32 => Property.Int, + TypeCode.Int64 => Property.Int, + TypeCode.UInt64 => Property.Int, + TypeCode.DateTime => Property.Date, + TypeCode.Boolean => Property.Bool, + TypeCode.Char => Property.Text, + TypeCode.SByte => null, + TypeCode.Byte => null, + TypeCode.Single => Property.Number, + TypeCode.Double => Property.Number, + TypeCode.Decimal => Property.Number, + TypeCode.Empty => null, + TypeCode.Object => null, + TypeCode.DBNull => null, + _ => null, + }; + + if (f is not null) + { + return f(name); + } + + // Handle arrays and collections + if (IsArrayOrCollection(actualType, out Type? elementType)) + { + return HandleCollectionType(elementType, name); + } + + throw new NotSupportedException($"Type {t.Name} not supported"); + } + + private static bool IsArrayOrCollection(Type type, out Type? elementType) + { + elementType = null; + + // Handle arrays + if (type.IsArray) + { + elementType = type.GetElementType(); + return elementType != null; + } + + // Handle generic collections (List, IEnumerable, etc.) + if (type.IsGenericType) + { + var genericTypeDef = type.GetGenericTypeDefinition(); + + // Check for common collection interfaces and types + if ( + genericTypeDef == typeof(IEnumerable<>) + || genericTypeDef == typeof(ICollection<>) + || genericTypeDef == typeof(IList<>) + || genericTypeDef == typeof(List<>) + || genericTypeDef == typeof(HashSet<>) + || genericTypeDef == typeof(ISet<>) + ) + { + elementType = type.GetGenericArguments()[0]; + return true; + } + } + + // Check if type implements IEnumerable + var enumerableInterface = type.GetInterfaces() + .FirstOrDefault(i => + i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>) + ); + + if (enumerableInterface != null) + { + elementType = enumerableInterface.GetGenericArguments()[0]; + return true; + } + + return false; + } + + private static Property HandleCollectionType(Type? elementType, string name) + { + if (elementType == null) + return null!; // or throw an exception + + // Handle special collection element types + if (elementType == typeof(Guid)) + { + return Property.UuidArray(name); // Assuming you have array-specific methods + } + + // Handle primitive collection element types + Func? f = Type.GetTypeCode(elementType) switch + { + TypeCode.String => Property.TextArray, + TypeCode.Int16 => Property.IntArray, + TypeCode.UInt16 => Property.IntArray, + TypeCode.Int32 => Property.IntArray, + TypeCode.UInt32 => Property.IntArray, + TypeCode.Int64 => Property.IntArray, + TypeCode.UInt64 => Property.IntArray, + TypeCode.DateTime => Property.DateArray, + TypeCode.Boolean => Property.BoolArray, + TypeCode.Char => Property.TextArray, + TypeCode.Single => Property.NumberArray, + TypeCode.Double => Property.NumberArray, + TypeCode.Decimal => Property.NumberArray, + _ => null, + }; + + return f!(name); + } +} + public static class DataType { - public static string Date { get; } = "date"; - public static string GeoCoordinate { get; } = "geo"; + public static string Text { get; } = "text"; + public static string TextArray { get; } = "text[]"; public static string Int { get; } = "int"; + public static string IntArray { get; } = "int[]"; public static string Bool { get; } = "boolean"; - public static string List { get; } = "list"; + public static string BoolArray { get; } = "boolean[]"; public static string Number { get; } = "number"; - public static string Object { get; } = "object"; - public static string PhoneNumber { get; } = "phone"; - public static string Text { get; } = "text"; + public static string NumberArray { get; } = "number[]"; + public static string Date { get; } = "date"; + public static string DateArray { get; } = "date[]"; public static string Uuid { get; } = "uuid"; - - public static string Reference(string property) => property.Capitalize(); + public static string UuidArray { get; } = "uuid[]"; + public static string GeoCoordinate { get; } = "geoCoordinates"; + public static string Blob { get; } = "blob"; + public static string PhoneNumber { get; } = "phone"; + public static string Object { get; } = "object"; + public static string ObjectArray { get; } = "object[]"; } -public class ReferenceProperty +public record ReferenceProperty { public required string Name { get; set; } public required string TargetCollection { get; set; } public static implicit operator Property(ReferenceProperty p) { - return new Property - { - Name = p.Name, - DataType = { DataType.Reference(p.TargetCollection) }, - }; + return new Property { Name = p.Name, DataType = { p.TargetCollection.Capitalize() } }; } } -public class Property +public partial record Property { public required string Name { get; set; } public IList DataType { get; set; } = new List(); @@ -41,72 +183,70 @@ public class Property public bool? IndexRangeFilters { get; set; } public bool? IndexSearchable { get; set; } - public static Property Text(string name) - { - return new Property { Name = name, DataType = { Models.DataType.Text } }; - } + public static Property Text(string name) => + new() { Name = name, DataType = { Models.DataType.Text } }; - public static Property Int(string name) - { - return new Property { Name = name, DataType = { Models.DataType.Int } }; - } + public static Property TextArray(string name) => + new() { Name = name, DataType = { Models.DataType.TextArray } }; - public static Property Date(string name) - { - return new Property { Name = name, DataType = { Models.DataType.Date } }; - } + public static Property Int(string name) => + new() { Name = name, DataType = { Models.DataType.Int } }; - public static Property Number(string name) - { - return new Property { Name = name, DataType = { Models.DataType.Number } }; - } + public static Property IntArray(string name) => + new() { Name = name, DataType = { Models.DataType.IntArray } }; - internal static Property Bool(string name) - { - return new Property { Name = name, DataType = { Models.DataType.Bool } }; - } + public static Property Bool(string name) => + new() { Name = name, DataType = { Models.DataType.Bool } }; - public static ReferenceProperty Reference(string name, string targetCollection) - { - return new ReferenceProperty { Name = name, TargetCollection = targetCollection }; - } + public static Property BoolArray(string name) => + new() { Name = name, DataType = { Models.DataType.BoolArray } }; - private static Property For(Type t, string name) - { - Func? f = Type.GetTypeCode(t) switch - { - TypeCode.String => Text, - TypeCode.Int16 => Int, - TypeCode.UInt16 => Int, - TypeCode.Int32 => Int, - TypeCode.UInt32 => Int, - TypeCode.Int64 => Int, - TypeCode.UInt64 => Int, - TypeCode.DateTime => Date, - TypeCode.Boolean => Bool, - TypeCode.Char => Text, - TypeCode.SByte => null, - TypeCode.Byte => null, - TypeCode.Single => Number, - TypeCode.Double => Number, - TypeCode.Decimal => Number, - TypeCode.Empty => null, - TypeCode.Object => null, - TypeCode.DBNull => null, - _ => null, - }; + public static Property Number(string name) => + new() { Name = name, DataType = { Models.DataType.Number } }; - return f!(name); - } + public static Property NumberArray(string name) => + new() { Name = name, DataType = { Models.DataType.NumberArray } }; - public static Property For(string name) - { - return For(typeof(TField), name); - } + public static Property Date(string name) => + new() { Name = name, DataType = { Models.DataType.Date } }; + + public static Property DateArray(string name) => + new() { Name = name, DataType = { Models.DataType.DateArray } }; + + public static Property Uuid(string name) => + new() { Name = name, DataType = { Models.DataType.Uuid } }; + + public static Property UuidArray(string name) => + new() { Name = name, DataType = { Models.DataType.UuidArray } }; + + public static Property GeoCoordinate(string name) => + new() { Name = name, DataType = { Models.DataType.GeoCoordinate } }; + + public static Property Blob(string name) => + new() { Name = name, DataType = { Models.DataType.Blob } }; + + public static Property PhoneNumber(string name) => + new() { Name = name, DataType = { Models.DataType.PhoneNumber } }; + + public static Property Object(string name) => + new() { Name = name, DataType = { Models.DataType.Object } }; + + public static Property ObjectArray(string name) => + new() { Name = name, DataType = { Models.DataType.ObjectArray } }; + + public static ReferenceProperty Reference(string name, string targetCollection) => + new() { Name = name, TargetCollection = targetCollection }; + + public static Property For(string name) => PropertyHelper.For(typeof(TField), name); // Extract collection properties from type specified by TData. public static IList FromCollection() { - return [.. typeof(TData).GetProperties().Select(x => For(x.PropertyType, x.Name))]; + return + [ + .. typeof(TData) + .GetProperties() + .Select(x => PropertyHelper.For(x.PropertyType, x.Name)), + ]; } } diff --git a/src/Weaviate.Client/Models/WeaviateObject.cs b/src/Weaviate.Client/Models/WeaviateObject.cs index 23c44688..d2331983 100644 --- a/src/Weaviate.Client/Models/WeaviateObject.cs +++ b/src/Weaviate.Client/Models/WeaviateObject.cs @@ -51,13 +51,15 @@ public partial record WeaviateObject public NamedVectors Vectors { get; set; } = new NamedVectors(); public T? As() + where T : new() { - return UnmarshallProperties(Properties); + return ObjectHelper.UnmarshallProperties(Properties); } public void Do(Action action) + where T : new() { - var data = UnmarshallProperties(Properties); + var data = As(); if (data is not null) { action(data); @@ -70,8 +72,9 @@ public void Do(Action action) } public TResult? Get(Func func) + where TSource : new() { - var data = UnmarshallProperties(Properties); + var data = ObjectHelper.UnmarshallProperties(Properties); if (data is not null) { return func(data); @@ -83,61 +86,4 @@ public void Do(Action action) { return Get(func); } - - internal static T? UnmarshallProperties(IDictionary dict) - { - if (dict == null) - throw new ArgumentNullException(nameof(dict)); - - // Create an instance of T using the default constructor - var props = Activator.CreateInstance(); - - if (props is IDictionary target) - { - foreach (var kvp in dict) - { - if (kvp.Value is IDictionary subDict) - { - object? v = UnmarshallProperties(subDict); - - target[kvp.Key.Capitalize()] = v ?? subDict; - } - else - { - target[kvp.Key.Capitalize()] = kvp.Value; - } - } - return props; - } - - var type = typeof(T); - var properties = type.GetProperties(); - - foreach (var property in properties) - { - var matchingKey = dict.Keys.FirstOrDefault(k => - string.Equals(k, property.Name, StringComparison.OrdinalIgnoreCase) - ); - - if (matchingKey != null) - { - var value = dict[matchingKey]; - if (value != null) - { - try - { - var convertedValue = Convert.ChangeType(value, property.PropertyType); - property.SetValue(props, convertedValue); - } - catch - { - // Skip if conversion fails - continue; - } - } - } - } - - return props; - } } diff --git a/src/Weaviate.Client/gRPC/Client.cs b/src/Weaviate.Client/gRPC/Client.cs index 84d5304d..3be4ef83 100644 --- a/src/Weaviate.Client/gRPC/Client.cs +++ b/src/Weaviate.Client/gRPC/Client.cs @@ -42,17 +42,19 @@ private static IList MakeListValue(ListValue list) switch (list.KindCase) { case ListValue.KindOneofCase.BoolValues: - return list.BoolValues.Values; + return list.BoolValues.Values.ToArray(); case ListValue.KindOneofCase.ObjectValues: - return list.ObjectValues.Values.Select(v => MakeNonRefs(v)).ToList(); + return list.ObjectValues.Values.Select(v => MakeNonRefs(v)).ToArray(); case ListValue.KindOneofCase.DateValues: - return list.DateValues.Values; // TODO Parse dates here? + return list.DateValues.Values.Select(v => DateTime.Parse(v)).ToArray(); case ListValue.KindOneofCase.UuidValues: - return list.UuidValues.Values.Select(v => Guid.Parse(v)).ToList(); + return list.UuidValues.Values.Select(v => Guid.Parse(v)).ToArray(); case ListValue.KindOneofCase.TextValues: return list.TextValues.Values; - case ListValue.KindOneofCase.IntValues: // TODO Decode list.IntValues according to docs - case ListValue.KindOneofCase.NumberValues: // TODO Decode list.NumberValues according to docs + case ListValue.KindOneofCase.IntValues: + return list.IntValues.Values.FromByteString().ToArray(); + case ListValue.KindOneofCase.NumberValues: + return list.NumberValues.Values.FromByteString().ToArray(); case ListValue.KindOneofCase.None: default: return new List { }; @@ -105,7 +107,10 @@ private static ExpandoObject MakeNonRefs(Properties result) eo[r.Key] = r.Value.IntValue; break; case Value.KindOneofCase.GeoValue: - eo[r.Key] = r.Value.GeoValue.ToString(); + eo[r.Key] = new Models.GeoCoordinate( + r.Value.GeoValue.Latitude, + r.Value.GeoValue.Longitude + ); break; case Value.KindOneofCase.BlobValue: eo[r.Key] = r.Value.BlobValue; From f8cdc24779a3d763a04f2f1ad7c32d304e0a2ea5 Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Fri, 6 Jun 2025 20:33:12 +0200 Subject: [PATCH 2/3] Support array types in filters --- .../Integration/Datasets.cs | 155 +++++++++++++++ .../Integration/Filters.cs | 184 +++++++++++++++++- src/Weaviate.Client.Tests/Unit/Filters.cs | 2 +- src/Weaviate.Client.Tests/Unit/Properties.cs | 52 ++--- src/Weaviate.Client/Models/Filter.cs | 3 + src/Weaviate.Client/Models/Vectorizer.cs | 2 + 6 files changed, 363 insertions(+), 35 deletions(-) diff --git a/src/Weaviate.Client.Tests/Integration/Datasets.cs b/src/Weaviate.Client.Tests/Integration/Datasets.cs index ee7c4f67..332d3a40 100644 --- a/src/Weaviate.Client.Tests/Integration/Datasets.cs +++ b/src/Weaviate.Client.Tests/Integration/Datasets.cs @@ -4,6 +4,161 @@ namespace Weaviate.Client.Tests.Integration; public partial class BasicTests { + public class DatasetFilterArrayTypes : TheoryData + { + public static Dictionary Cases = new() + { + ["Test 1"] = (Filter.Property("texts").Like("*nana"), new int[] { 1 }), + ["Test 2"] = (Filter.Property("texts").Equal("banana"), new int[] { 1 }), + ["Test 3"] = (Filter.Property("ints").Equal(3), new int[] { 1 }), + ["Test 4"] = (Filter.Property("ints").GreaterThanEqual(3), new int[] { 1, 2 }), + ["Test 5"] = (Filter.Property("floats").Equal(3), new int[] { 1 }), + ["Test 6"] = (Filter.Property("floats").LessThanEqual(3), new int[] { 0, 1 }), + }; + + public DatasetFilterArrayTypes() + : base(Cases.Keys) { } + } + + // Define test constants + private static readonly DateTime NOW = DateTime.UtcNow; + private static readonly DateTime LATER = NOW.AddHours(1); + private static readonly DateTime MUCH_LATER = NOW.AddDays(1); + private static readonly Guid UUID1 = Guid.NewGuid(); + private static readonly Guid UUID2 = Guid.NewGuid(); + private static readonly Guid UUID3 = Guid.NewGuid(); + + public class DatasetFilterContains : TheoryData + { + public static Dictionary Cases = new() + { + ["ContainsAny ints 1,4"] = ( + Filter.Property("ints").ContainsAny([1, 4]), + new int[] { 0, 3 } + ), + ["ContainsAny ints 1.0,4"] = ( + Filter.Property("ints").ContainsAny([1.0, 4]), + new int[] { 0, 3 } + ), + ["ContainsAny ints 10"] = (Filter.Property("ints").ContainsAny([10]), new int[] { }), + ["ContainsAny int 1"] = (Filter.Property("int").ContainsAny([1]), new int[] { 0, 1 }), + ["ContainsAny text test"] = ( + Filter.Property("text").ContainsAny(["test"]), + new int[] { 0, 1 } + ), + ["ContainsAny text real,deal"] = ( + Filter.Property("text").ContainsAny(["real", "deal"]), + new int[] { 1, 2, 3 } + ), + ["ContainsAny texts test"] = ( + Filter.Property("texts").ContainsAny(["test"]), + new int[] { 0, 1 } + ), + ["ContainsAny texts real,deal"] = ( + Filter.Property("texts").ContainsAny(["real", "deal"]), + new int[] { 1, 2, 3 } + ), + ["ContainsAny float 2.0"] = ( + Filter.Property("float").ContainsAny([2.0]), + new int[] { } + ), + ["ContainsAny float 2"] = (Filter.Property("float").ContainsAny([2]), new int[] { }), + ["ContainsAny float 8"] = (Filter.Property("float").ContainsAny([8]), new int[] { 3 }), + ["ContainsAny float 8.0"] = ( + Filter.Property("float").ContainsAny([8.0]), + new int[] { 3 } + ), + ["ContainsAny floats 2.0"] = ( + Filter.Property("floats").ContainsAny([2.0]), + new int[] { 0, 1 } + ), + ["ContainsAny floats 0.4,0.7"] = ( + Filter.Property("floats").ContainsAny([0.4, 0.7]), + new int[] { 0, 1, 3 } + ), + ["ContainsAny floats 2"] = ( + Filter.Property("floats").ContainsAny([2]), + new int[] { 0, 1 } + ), + ["ContainsAny bools true,false"] = ( + Filter.Property("bools").ContainsAny([true, false]), + new int[] { 0, 1, 3 } + ), + ["ContainsAny bools false"] = ( + Filter.Property("bools").ContainsAny([false]), + new int[] { 0, 1 } + ), + ["ContainsAny bool true"] = ( + Filter.Property("bool").ContainsAny([true]), + new int[] { 0, 1, 3 } + ), + ["ContainsAll ints 1,4"] = ( + Filter.Property("ints").ContainsAll([1, 4]), + new int[] { 0 } + ), + ["ContainsAll text real,test"] = ( + Filter.Property("text").ContainsAll(["real", "test"]), + new int[] { 1 } + ), + ["ContainsAll texts real,test"] = ( + Filter.Property("texts").ContainsAll(["real", "test"]), + new int[] { 1 } + ), + ["ContainsAll floats 0.7,2"] = ( + Filter.Property("floats").ContainsAll([0.7, 2]), + new int[] { 1 } + ), + ["ContainsAll bools true,false"] = ( + Filter.Property("bools").ContainsAll([true, false]), + new int[] { 0 } + ), + ["ContainsAll bool true,false"] = ( + Filter.Property("bool").ContainsAll([true, false]), + new int[] { } + ), + ["ContainsAll bool true"] = ( + Filter.Property("bool").ContainsAll([true]), + new int[] { 0, 1, 3 } + ), + ["ContainsAny dates now,much_later"] = ( + Filter.Property("dates").ContainsAny([NOW, MUCH_LATER]), + new int[] { 0, 1, 3 } + ), + ["ContainsAny dates now"] = ( + Filter.Property("dates").ContainsAny([NOW]), + new int[] { 0, 1 } + ), + ["Equal date now"] = (Filter.Property("date").Equal(NOW), new int[] { 0 }), + ["GreaterThan date now"] = ( + Filter.Property("date").GreaterThan(NOW), + new int[] { 1, 3 } + ), + ["ContainsAll uuids uuid2,uuid1"] = ( + Filter.Property("uuids").ContainsAll([UUID2, UUID1]), + new int[] { 0, 3 } + ), + ["ContainsAny uuids uuid2,uuid1"] = ( + Filter.Property("uuids").ContainsAny([UUID2, UUID1]), + new int[] { 0, 1, 3 } + ), + ["ContainsAny uuid uuid3"] = ( + Filter.Property("uuid").ContainsAny([UUID3]), + new int[] { } + ), + ["ContainsAny uuid uuid1"] = ( + Filter.Property("uuid").ContainsAny([UUID1]), + new int[] { 0 } + ), + ["ContainsAny _id uuid1,uuid3"] = ( + Filter.Property("_id").ContainsAny([UUID1, UUID3]), + new int[] { 0, 2 } + ), + }; + + public DatasetFilterContains() + : base(Cases.Keys) { } + } + public class DatasetRefCountFilter : TheoryData { public static Dictionary Cases => diff --git a/src/Weaviate.Client.Tests/Integration/Filters.cs b/src/Weaviate.Client.Tests/Integration/Filters.cs index 15f7606a..0b8c09c4 100644 --- a/src/Weaviate.Client.Tests/Integration/Filters.cs +++ b/src/Weaviate.Client.Tests/Integration/Filters.cs @@ -276,15 +276,6 @@ public async Task TimeFilterContains() Assert.True(objs.All(obj => obj.ID != null && expectedUuids.Contains(obj.ID.Value))); } - public static Dictionary Cases => - new() - { - ["IdEquals"] = Filter.ID.Equal(_reusableUuids[0]), - ["IdContainsAny"] = Filter.ID.ContainsAny([_reusableUuids[0]]), - ["IdNotEqual"] = Filter.ID.NotEqual(_reusableUuids[1]), - ["IdWithProperty(_id)Equal"] = Filter.Property("_id").Equal(_reusableUuids[0]), - }; - [Theory] [ClassData(typeof(DatasetTimeFilter))] public async Task TimeFiltering(string key) @@ -324,4 +315,179 @@ public async Task TimeFiltering(string key) ); Assert.True(objs.All(obj => obj.ID != null && expectedUuids.Contains(obj.ID.Value))); } + + [Theory] + [ClassData(typeof(DatasetFilterArrayTypes))] + public async Task FilterArrayTypes(string key) + { + (Filter filter, int[] results) = DatasetFilterArrayTypes.Cases[key]; + + // Arrange + var collection = await CollectionFactory( + vectorConfig: new Dictionary + { + { + "default", + new VectorConfig { Vectorizer = Vectorizer.None, VectorIndexType = "hnsw" } + }, + }, + properties: + [ + Property.TextArray("texts"), + Property.IntArray("ints"), + Property.NumberArray("floats"), + ] + ); + + var uuids = new[] + { + await collection.Data.Insert( + new + { + texts = new[] { "an", "apple" }, + ints = new[] { 1, 2 }, + floats = new[] { 1.0, 2.0 }, + } + ), + await collection.Data.Insert( + new + { + texts = new[] { "a", "banana" }, + ints = new[] { 2, 3 }, + floats = new[] { 2.0, 3.0 }, + } + ), + await collection.Data.Insert( + new + { + texts = new[] { "a", "text" }, + ints = new[] { 4, 5 }, + floats = new[] { 4.0, 5.0 }, + } + ), + }; + + // Act + var objects = await collection.Query.List(filter: filter); + + // Assert + Assert.Equal(results.Length, objects.Count()); + + var expectedUuids = results.Select(result => uuids[result]).ToHashSet(); + Assert.True(objects.All(obj => expectedUuids.Contains(obj.ID!.Value))); + } + + [Theory] + [ClassData(typeof(DatasetFilterContains))] + public async Task FilterContains(string test) + { + (Filter filter, int[] results) = DatasetFilterContains.Cases[test]; + + // Arrange + var collection = await CollectionFactory( + vectorConfig: new Dictionary + { + { + "default", + new VectorConfig { Vectorizer = Vectorizer.None, VectorIndexType = "hnsw" } + }, + }, + properties: + [ + Property.Text("text"), + Property.TextArray("texts"), + Property.Int("int"), + Property.IntArray("ints"), + Property.Number("float"), + Property.NumberArray("floats"), + Property.Bool("bool"), + Property.BoolArray("bools"), + Property.DateArray("dates"), + Property.Date("date"), + Property.UuidArray("uuids"), + Property.Uuid("uuid"), + ] + ); + + var uuids = new[] + { + await collection.Data.Insert( + new + { + text = "this is a test", + texts = new[] { "this", "is", "a", "test" }, + @int = 1, + ints = new[] { 1, 2, 4 }, + @float = 0.5, + floats = new[] { 0.4, 0.9, 2.0 }, + @bool = true, + bools = new[] { true, false }, + dates = new[] { NOW, LATER, MUCH_LATER }, + date = NOW, + uuids = new[] { UUID1, UUID3, UUID2 }, + uuid = UUID1, + }, + id: UUID1 + ), + await collection.Data.Insert( + new + { + text = "this is not a real test", + texts = new[] { "this", "is", "not", "a", "real", "test" }, + @int = 1, + ints = new[] { 5, 6, 9 }, + @float = 0.3, + floats = new[] { 0.1, 0.7, 2.0 }, + @bool = true, + bools = new[] { false, false }, + dates = new[] { NOW, NOW, MUCH_LATER }, + date = LATER, + uuids = new[] { UUID2, UUID2 }, + uuid = UUID2, + }, + id: UUID2 + ), + await collection.Data.Insert( + new + { + text = "real deal", + texts = new[] { "real", "deal" }, + @int = 3, + ints = new int[0], + floats = new double[0], + @bool = false, + bools = new bool[0], + dates = new DateTime[0], + uuids = new Guid[0], + }, + id: UUID3 + ), + await collection.Data.Insert( + new + { + text = "not real deal", + texts = new[] { "not", "real", "deal" }, + @int = 4, + ints = new[] { 4 }, + @float = 8.0, + floats = new[] { 0.7 }, + @bool = true, + bools = new[] { true }, + dates = new[] { MUCH_LATER }, + date = MUCH_LATER, + uuids = new[] { UUID1, UUID2 }, + uuid = UUID2, + } + ), + }; + + // Act + var objects = await collection.Query.List(filter: filter); + + // Assert + Assert.Equal(results.Length, objects.Count()); + + var expectedUuids = results.Select(result => uuids[result]).ToHashSet(); + Assert.True(objects.All(obj => expectedUuids.Contains(obj.ID!.Value))); + } } diff --git a/src/Weaviate.Client.Tests/Unit/Filters.cs b/src/Weaviate.Client.Tests/Unit/Filters.cs index 05113efc..63efe045 100644 --- a/src/Weaviate.Client.Tests/Unit/Filters.cs +++ b/src/Weaviate.Client.Tests/Unit/Filters.cs @@ -35,7 +35,7 @@ string[] unexpectedMethodList } [Fact] - public void FilterByReferenceDoesNotChangePreviousFilter() + public void FilterByReferencChainingChangePreviousFilterBecauseRefs() { // Arrange var f1 = Filter.Reference("ref"); diff --git a/src/Weaviate.Client.Tests/Unit/Properties.cs b/src/Weaviate.Client.Tests/Unit/Properties.cs index 79e57c1a..2aa9a4db 100644 --- a/src/Weaviate.Client.Tests/Unit/Properties.cs +++ b/src/Weaviate.Client.Tests/Unit/Properties.cs @@ -4,36 +4,38 @@ namespace Weaviate.Client.Tests; public partial class UnitTests { - public static IEnumerable CaseKeys => Cases.Keys.Select(k => new object[] { k }); + public static TheoryData PropertyCasesKeys => [.. PropertyCases.Keys]; - private static Dictionary, Func)> Cases = - new() - { - ["Text"] = (Property.Text, Property.For), - ["TextArray"] = (Property.TextArray, Property.For), - ["Int"] = (Property.Int, Property.For), - ["IntArray"] = (Property.IntArray, Property.For), - ["Bool"] = (Property.Bool, Property.For), - ["BoolArray"] = (Property.BoolArray, Property.For), - ["Number"] = (Property.Number, Property.For), - ["NumberArray"] = (Property.NumberArray, Property.For), - ["Date"] = (Property.Date, Property.For), - ["DateArray"] = (Property.DateArray, Property.For), - ["Uuid"] = (Property.Uuid, Property.For), - ["UuidArray"] = (Property.UuidArray, Property.For), - ["Geo"] = (Property.GeoCoordinate, Property.For), - ["Phone"] = (Property.PhoneNumber, Property.For), - // TODO Add support for the properties below - // ["Blob"] = (Property.Blob, Property.For), - // ["Object"] = (Property.Object, Property.For), - // ["ObjectArray"] = (Property.ObjectArray, Property.For), - }; + private static Dictionary< + string, + (Func, Func) + > PropertyCases = new() + { + ["Text"] = (Property.Text, Property.For), + ["TextArray"] = (Property.TextArray, Property.For), + ["Int"] = (Property.Int, Property.For), + ["IntArray"] = (Property.IntArray, Property.For), + ["Bool"] = (Property.Bool, Property.For), + ["BoolArray"] = (Property.BoolArray, Property.For), + ["Number"] = (Property.Number, Property.For), + ["NumberArray"] = (Property.NumberArray, Property.For), + ["Date"] = (Property.Date, Property.For), + ["DateArray"] = (Property.DateArray, Property.For), + ["Uuid"] = (Property.Uuid, Property.For), + ["UuidArray"] = (Property.UuidArray, Property.For), + ["Geo"] = (Property.GeoCoordinate, Property.For), + ["Phone"] = (Property.PhoneNumber, Property.For), + // TODO Add support for the properties below + // ["Blob"] = (Property.Blob, Property.For), + // ["Object"] = (Property.Object, Property.For), + // ["ObjectArray"] = (Property.ObjectArray, Property.For), + }; [Theory] - [MemberData(nameof(CaseKeys))] + [MemberData(nameof(PropertyCasesKeys))] public void Properties(string test) { - var (f1, f2) = Cases[test]; + var (f1, f2) = PropertyCases[test]; var (p1, p2) = (f1(test), f2(test)); Assert.Equivalent(p1, p2); diff --git a/src/Weaviate.Client/Models/Filter.cs b/src/Weaviate.Client/Models/Filter.cs index d0ceee15..8eaedaf6 100644 --- a/src/Weaviate.Client/Models/Filter.cs +++ b/src/Weaviate.Client/Models/Filter.cs @@ -167,12 +167,15 @@ internal Filter WithValue(T value) }, IEnumerable v => f => f.ValueBooleanArray = new BooleanArray { Values = { v } }, + IEnumerable v => f => + f.ValueIntArray = new IntArray { Values = { v.Select(Convert.ToInt64) } }, IEnumerable v => f => f.ValueIntArray = new IntArray { Values = { v } }, IEnumerable v => f => f.ValueNumberArray = new NumberArray { Values = { v } }, IEnumerable v => f => f.ValueTextArray = new TextArray { Values = { v } }, IEnumerable v => f => f.ValueTextArray = new TextArray { Values = { v.Select(g => g.ToString()) } }, + // TODO Perhaps add a case handling generic IEnumerable _ => throw new WeaviateException( $"Unsupported type '{typeof(T).Name}' for filter value. Check the documentation for supported filter value types." ), diff --git a/src/Weaviate.Client/Models/Vectorizer.cs b/src/Weaviate.Client/Models/Vectorizer.cs index c62ab104..27f5c96f 100644 --- a/src/Weaviate.Client/Models/Vectorizer.cs +++ b/src/Weaviate.Client/Models/Vectorizer.cs @@ -29,6 +29,8 @@ public static implicit operator Dictionary(Vectorizer vectorizer return new Dictionary { [vectorizer.Name] = vectorizer.Configuration }; } + public static readonly Vectorizer None = new("none", new { }); + public static Vectorizer Text2VecContextionary(bool vectorizeClassName = false) { return new Vectorizer("text2vec-contextionary", new { VectorizeClassName = false }); From 4617809fabe7d879a3d650716ad4c72c97d1c441 Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Tue, 10 Jun 2025 15:37:36 +0200 Subject: [PATCH 3/3] Added new test cases for batch insertion with array property types. Extras: - Simplified object retrieval in FetchObjectByID method to return a single object. - Updated tests to assert against the new object structure. - Enhanced batch property handling to support array types in DataClient. - Improved extension methods for better stream handling of native types. --- src/Example/Program.cs | 3 +- src/Weaviate.Client.Tests/Integration/Data.cs | 11 +- .../Integration/Filters.cs | 7 +- .../Integration/NearText.cs | 3 +- .../Integration/NearVector.cs | 7 +- .../Integration/Properties.cs | 125 ++++++-- src/Weaviate.Client.Tests/Unit/_Unit.cs | 130 +++++++- src/Weaviate.Client/DataClient.cs | 302 +++++++++++++++--- src/Weaviate.Client/Extensions.cs | 10 +- src/Weaviate.Client/QueryClient.cs | 16 +- 10 files changed, 516 insertions(+), 98 deletions(-) diff --git a/src/Example/Program.cs b/src/Example/Program.cs index 4e9c3e18..ab77f30d 100644 --- a/src/Example/Program.cs +++ b/src/Example/Program.cs @@ -144,8 +144,7 @@ static async Task Main() { var fetched = await collection.Query.FetchObjectByID(id: id2); Console.WriteLine( - "Cat retrieved via gRPC matches: " - + ((fetched?.Objects.First().ID ?? Guid.Empty) == id2) + "Cat retrieved via gRPC matches: " + ((fetched?.ID ?? Guid.Empty) == id2) ); } diff --git a/src/Weaviate.Client.Tests/Integration/Data.cs b/src/Weaviate.Client.Tests/Integration/Data.cs index f0801963..ada1adc6 100644 --- a/src/Weaviate.Client.Tests/Integration/Data.cs +++ b/src/Weaviate.Client.Tests/Integration/Data.cs @@ -41,18 +41,15 @@ public async Task ObjectCreation() // Assert object exists var retrieved = await collectionClient.Query.FetchObjectByID(id); - var objects = retrieved.Objects.ToList(); Assert.NotNull(retrieved); - Assert.Single(objects); - Assert.Equal(id, objects[0].ID); - Assert.Equal("TestObject", objects[0].Properties["name"]); - Assert.Equal("TestObject", objects[0].As()?.Name); + Assert.Equal(id, retrieved.ID); + Assert.Equal("TestObject", retrieved.Properties["name"]); + Assert.Equal("TestObject", retrieved.As()?.Name); // Delete after usage await collectionClient.Data.Delete(id); retrieved = await collectionClient.Query.FetchObjectByID(id); - Assert.NotNull(retrieved.Objects); - Assert.Empty(retrieved.Objects); + Assert.Null(retrieved); } } diff --git a/src/Weaviate.Client.Tests/Integration/Filters.cs b/src/Weaviate.Client.Tests/Integration/Filters.cs index 0b8c09c4..2f2c3275 100644 --- a/src/Weaviate.Client.Tests/Integration/Filters.cs +++ b/src/Weaviate.Client.Tests/Integration/Filters.cs @@ -42,7 +42,7 @@ public async Task FilteringWithMetadataDates() ); // Act - var objA1 = objsA1.First(); + var objA1 = objsA1; Assert.NotNull(objA1.Metadata.CreationTime); Assert.Equal(DateTimeKind.Utc, objA1.Metadata.CreationTime.Value.Kind); @@ -261,10 +261,7 @@ public async Task TimeFilterContains() // Act var objects = await collection.Query.List( filter: Filter.CreationTime.ContainsAny( - [ - obj2.First().Metadata.CreationTime!.Value, - obj3.First().Metadata.CreationTime!.Value, - ] + [obj2.Metadata.CreationTime!.Value, obj3.Metadata.CreationTime!.Value] ) ); diff --git a/src/Weaviate.Client.Tests/Integration/NearText.cs b/src/Weaviate.Client.Tests/Integration/NearText.cs index 64c1a7f6..16142300 100644 --- a/src/Weaviate.Client.Tests/Integration/NearText.cs +++ b/src/Weaviate.Client.Tests/Integration/NearText.cs @@ -99,11 +99,10 @@ public async Task NearTextGroupBySearch() Assert.Equal(2, retrieved.Objects.Count()); Assert.Equal(2, retrieved.Groups.Count()); - var result = await collectionClient.Query.FetchObjectByID( + var obj = await collectionClient.Query.FetchObjectByID( guids[3], metadata: new MetadataQuery(Vectors: ["default"]) ); - var obj = result.Objects.First(); Assert.NotNull(obj); Assert.Equal(guids[3], obj.ID); diff --git a/src/Weaviate.Client.Tests/Integration/NearVector.cs b/src/Weaviate.Client.Tests/Integration/NearVector.cs index 2bc75562..0e675851 100644 --- a/src/Weaviate.Client.Tests/Integration/NearVector.cs +++ b/src/Weaviate.Client.Tests/Integration/NearVector.cs @@ -8,7 +8,10 @@ public partial class BasicTests public async Task NearVectorSearch() { // Arrange - var collectionClient = await CollectionFactory("", "Test collection description"); + var collectionClient = await CollectionFactory( + "NearVectorSearch3vecs", + "Test collection description" + ); // Act await collectionClient.Data.Insert( @@ -26,6 +29,8 @@ await collectionClient.Data.Insert( vectors: new NamedVectors { { "default", 0.5f, 0.6f, 0.7f } } ); + var objs = await collectionClient.Query.List(metadata: MetadataOptions.Vector); + // Assert var retrieved = await collectionClient.Query.NearVector([0.1f, 0.2f, 0.3f]); Assert.NotNull(retrieved); diff --git a/src/Weaviate.Client.Tests/Integration/Properties.cs b/src/Weaviate.Client.Tests/Integration/Properties.cs index e14c171e..ada52643 100644 --- a/src/Weaviate.Client.Tests/Integration/Properties.cs +++ b/src/Weaviate.Client.Tests/Integration/Properties.cs @@ -4,27 +4,6 @@ namespace Weaviate.Client.Tests.Integration; public partial class BasicTests { - public record TestProperties - { - public string? TestText { get; set; } - public string[]? TestTextArray { get; set; } - public int? TestInt { get; set; } - public int[]? TestIntArray { get; set; } - public bool? TestBool { get; set; } - public bool[]? TestBoolArray { get; set; } - public double? TestNumber { get; set; } - public double[]? TestNumberArray { get; set; } - public DateTime? TestDate { get; set; } - public DateTime[]? TestDateArray { get; set; } - public Guid? TestUuid { get; set; } - public Guid[]? TestUuidArray { get; set; } - public GeoCoordinate? TestGeo { get; set; } - // // public byte[]? TestBlob { get; set; } - // // public PhoneNumber? TestPhone { get; set; } - // // public object? TestObject { get; set; } - // // public object? TestObjectArray { get; set; } - } - [Fact] public async Task AllPropertiesSaveRetrieve() { @@ -76,12 +55,110 @@ public async Task AllPropertiesSaveRetrieve() var id = await c.Data.Insert(testData); // 3. Retrieve the object and confirm all properties match - var retrieved = await c.Query.FetchObjectByID(id); - - var obj = retrieved.Objects.First(); + var obj = await c.Query.FetchObjectByID(id); var concreteObj = obj.As(); Assert.Equivalent(testData, concreteObj); } + + [Fact] + public async Task Test_BatchInsert_WithArrays() + { + Property[] props = + [ + Property.Text("testText"), + Property.TextArray("testTextArray"), + Property.Int("testInt"), + Property.IntArray("testIntArray"), + Property.Bool("testBool"), + Property.BoolArray("testBoolArray"), + Property.Number("testNumber"), + Property.NumberArray("testNumberArray"), + Property.Date("testDate"), + Property.DateArray("testDateArray"), + Property.Uuid("testUuid"), + Property.UuidArray("testUuidArray"), + Property.GeoCoordinate("testGeo"), + // Property.Blob("testBlob"), + // Property.PhoneNumber("testPhone"), + // Property.Object("testObject"), + // Property.ObjectArray("testObjectArray"), + ]; + + // 1. Create collection + var c = await CollectionFactory( + description: "Testing collection properties", + properties: props + ); + + // 2. Create an object with values for all properties + var testData = new[] + { + new TestProperties + { + TestText = "dummyText1", + TestTextArray = new[] { "dummyTextArray11", "dummyTextArray21" }, + TestInt = 123, + TestIntArray = new[] { 1, 2, 3 }, + TestBool = true, + TestBoolArray = new[] { true, false }, + TestNumber = 456.789, + TestNumberArray = new[] { 4.5, 6.7 }, + TestDate = DateTime.Now.AddDays(-1), + TestDateArray = new[] { DateTime.Now.AddDays(-2), DateTime.Now.AddDays(-3) }, + TestUuid = Guid.NewGuid(), + TestUuidArray = new[] { Guid.NewGuid(), Guid.NewGuid() }, + TestGeo = new GeoCoordinate(12.345f, 67.890f), + }, + new TestProperties + { + TestText = "dummyText", + TestTextArray = new[] { "dummyTextArray1", "dummyTextArray2" }, + TestInt = 456, + TestIntArray = new[] { 4, 5, 6 }, + TestBool = true, + TestBoolArray = new bool[] { }, + TestNumber = 789.987, + TestNumberArray = new[] { 5.6, 7.8 }, + TestDate = DateTime.Now.AddDays(-2), + TestDateArray = new[] { DateTime.Now.AddDays(-3), DateTime.Now.AddDays(-4) }, + TestUuid = Guid.NewGuid(), + TestUuidArray = new[] { Guid.NewGuid(), Guid.NewGuid() }, + TestGeo = new GeoCoordinate(23.456f, 78.910f), + }, + new TestProperties + { + TestText = "dummyText", + TestTextArray = new[] { "dummyTextArray1", "dummyTextArray2" }, + TestInt = 345, + TestIntArray = new[] { 3, 4, 6 }, + TestBool = true, + TestNumber = 567.897, + TestNumberArray = new[] { 6.7, 8.9 }, + TestDate = DateTime.Now.AddDays(+1), + TestDateArray = new[] { DateTime.Now.AddDays(+2), DateTime.Now.AddDays(+3) }, + TestUuid = Guid.NewGuid(), + TestUuidArray = new[] { Guid.NewGuid(), Guid.NewGuid() }, + TestGeo = new GeoCoordinate(34.567f, 98.765f), + }, + }; + + var response = await c.Data.InsertMany(batcher => + { + testData.ToList().ForEach(d => batcher(d)); + }); + + // 3. Retrieve the object and confirm all properties match + foreach (var r in response) + { + var obj = await c.Query.FetchObjectByID(r.ID!.Value); + + Assert.NotNull(obj); + + var concreteObj = obj.As(); + + Assert.Equivalent(testData[r.Index], concreteObj); + } + } } diff --git a/src/Weaviate.Client.Tests/Unit/_Unit.cs b/src/Weaviate.Client.Tests/Unit/_Unit.cs index 00081ae8..d6e6f506 100644 --- a/src/Weaviate.Client.Tests/Unit/_Unit.cs +++ b/src/Weaviate.Client.Tests/Unit/_Unit.cs @@ -4,6 +4,27 @@ namespace Weaviate.Client.Tests; +public record TestProperties +{ + public string? TestText { get; set; } + public string[]? TestTextArray { get; set; } + public int? TestInt { get; set; } + public int[]? TestIntArray { get; set; } + public bool? TestBool { get; set; } + public bool[]? TestBoolArray { get; set; } + public double? TestNumber { get; set; } + public double[]? TestNumberArray { get; set; } + public DateTime? TestDate { get; set; } + public DateTime[]? TestDateArray { get; set; } + public Guid? TestUuid { get; set; } + public Guid[]? TestUuidArray { get; set; } + public GeoCoordinate? TestGeo { get; set; } + // // public byte[]? TestBlob { get; set; } + // // public PhoneNumber? TestPhone { get; set; } + // // public object? TestObject { get; set; } + // // public object? TestObjectArray { get; set; } +} + [Collection("Unit Tests")] public partial class UnitTests { @@ -117,10 +138,11 @@ public void TestBuildDynamicObject() [InlineData(typeof(string), true)] [InlineData(typeof(string[]), true)] [InlineData(typeof(DateTime), true)] + [InlineData(typeof(GeoCoordinate), true)] [InlineData(typeof(Object), false)] [InlineData(typeof(Object[]), false)] [InlineData(typeof(WeaviateObject), false)] - public void TestIsNativeTypeCheck(Type type, bool expected) + public void Test_IsNativeType_Check(Type type, bool expected) { // Arrange @@ -130,4 +152,110 @@ public void TestIsNativeTypeCheck(Type type, bool expected) // Assert Assert.Equal(expected, result); } + + [Fact] + public void Test_BatchProperties_Build() + { + var p = new TestProperties + { + TestText = "dummyText", + TestTextArray = new[] { "dummyTextArray1", "dummyTextArray2" }, + TestInt = 345, + TestIntArray = new[] { 3, 4, 6 }, + TestBool = true, + TestNumber = 567.897, + TestNumberArray = new[] { 6.7, 8.9 }, + TestDate = DateTime.Now.AddDays(+1), + TestDateArray = new[] { DateTime.Now.AddDays(+2), DateTime.Now.AddDays(+3) }, + TestUuid = Guid.NewGuid(), + TestUuidArray = new[] { Guid.NewGuid(), Guid.NewGuid() }, + TestGeo = new GeoCoordinate(34.567f, 98.765f), + }; + + var bp = ObjectHelper.BuildBatchProperties(p); + + // Verify all expected properties are present + Assert.Contains("TestText", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestInt", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestBool", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestNumber", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestDate", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestUuid", bp.NonRefProperties.Fields.Keys); + Assert.Contains("TestGeo", bp.NonRefProperties.Fields.Keys); + + // String properties + Assert.Equal(p.TestText, bp.NonRefProperties.Fields["TestText"].StringValue); + Assert.Equal( + p.TestTextArray.Length, + bp.TextArrayProperties.Where(p => p.PropName == "TestTextArray").First().Values.Count + ); + Assert.Equal( + p.TestTextArray[0], + bp.TextArrayProperties.Where(p => p.PropName == "TestTextArray").First().Values[0] + ); + Assert.Equal( + p.TestTextArray[1], + bp.TextArrayProperties.Where(p => p.PropName == "TestTextArray").First().Values[1] + ); + + // Integer properties + Assert.Equal( + Convert.ToDouble(p.TestInt), + bp.NonRefProperties.Fields["TestInt"].NumberValue + ); + Assert.Equal( + p.TestIntArray.Length, + bp.IntArrayProperties.Where(p => p.PropName == "TestIntArray").First().Values.Count + ); + Assert.Equal( + p.TestIntArray[0], + bp.IntArrayProperties.Where(p => p.PropName == "TestIntArray").First().Values[0] + ); + Assert.Equal( + p.TestIntArray[1], + bp.IntArrayProperties.Where(p => p.PropName == "TestIntArray").First().Values[1] + ); + Assert.Equal( + p.TestIntArray[2], + bp.IntArrayProperties.Where(p => p.PropName == "TestIntArray").First().Values[2] + ); + + // Boolean property + Assert.Equal(p.TestBool, bp.NonRefProperties.Fields["TestBool"].BoolValue); + + // Number properties + Assert.Equal(p.TestNumber, bp.NonRefProperties.Fields["TestNumber"].NumberValue); + + Assert.NotEmpty(bp.NumberArrayProperties[0].ValuesBytes); + + // Additional check for number array field name + Assert.Equal("TestNumberArray", bp.NumberArrayProperties[0].PropName); + + // Date properties + Assert.Equal( + p.TestDate.Value.ToUniversalTime().ToString("o"), + bp.NonRefProperties.Fields["TestDate"].StringValue + ); + var dateList = bp.TextArrayProperties.Single(p => p.PropName == "TestDateArray"); + Assert.Equal(p.TestDateArray.Length, dateList.Values.Count); + Assert.Equal(p.TestDateArray[0].ToUniversalTime().ToString("o"), dateList.Values[0]); + Assert.Equal(p.TestDateArray[1].ToUniversalTime().ToString("o"), dateList.Values[1]); + + // UUID properties + Assert.Equal(p.TestUuid.ToString(), bp.NonRefProperties.Fields["TestUuid"].StringValue); + var uuidList = bp.TextArrayProperties.Single(p => p.PropName == "TestUuidArray"); + Assert.Equal(p.TestUuidArray.Length, uuidList.Values.Count); + Assert.Equal(p.TestUuidArray[0].ToString(), uuidList.Values[0]); + Assert.Equal(p.TestUuidArray[1].ToString(), uuidList.Values[1]); + + // Geo property + Assert.Equal( + p.TestGeo.Latitude, + bp.NonRefProperties.Fields["TestGeo"].StructValue.Fields["latitude"].NumberValue + ); + Assert.Equal( + p.TestGeo.Longitude, + bp.NonRefProperties.Fields["TestGeo"].StructValue.Fields["longitude"].NumberValue + ); + } } diff --git a/src/Weaviate.Client/DataClient.cs b/src/Weaviate.Client/DataClient.cs index b35886e3..f6a5028f 100644 --- a/src/Weaviate.Client/DataClient.cs +++ b/src/Weaviate.Client/DataClient.cs @@ -259,52 +259,8 @@ System.Type targetType return obj; } -} - -public class DataClient -{ - private readonly CollectionClient _collectionClient; - private WeaviateClient _client => _collectionClient.Client; - private string _collectionName => _collectionClient.Name; - - internal DataClient(CollectionClient collectionClient) - { - _collectionClient = collectionClient; - } - - public static IDictionary[] MakeBeacons(params Guid[] guids) - { - return - [ - .. guids.Select(uuid => new Dictionary - { - { "beacon", $"weaviate://localhost/{uuid}" }, - }), - ]; - } - - // Helper method to convert C# objects to protobuf Values - public static Value ConvertToValue(object obj) - { - return obj switch - { - null => Value.ForNull(), - bool b => Value.ForBool(b), - int i => Value.ForNumber(i), - long l => Value.ForNumber(l), - float f => Value.ForNumber(f), - double d => Value.ForNumber(d), - decimal dec => Value.ForNumber((double)dec), - string s => Value.ForString(s), - DateTime dt => Value.ForString(dt.ToUniversalTime().ToString("o")), - Guid uuid => Value.ForString(uuid.ToString()), - // Dictionary dict => Value.ForStruct(CreateStructFromDictionary(dict)), - // IEnumerable enumerable => CreateListValue(enumerable), - _ => throw new ArgumentException($"Unsupported type: {obj.GetType()}"), - }; - } - private V1.BatchObject.Types.Properties BuildBatchProperties(TProps data) + internal static V1.BatchObject.Types.Properties BuildBatchProperties(TProps data) { var props = new V1.BatchObject.Types.Properties(); @@ -317,6 +273,11 @@ private V1.BatchObject.Types.Properties BuildBatchProperties(TProps data foreach (var propertyInfo in data.GetType().GetProperties()) { + if (propertyInfo is null) + { + continue; + } + if (!propertyInfo.CanRead) continue; // skip non-readable properties @@ -327,11 +288,204 @@ private V1.BatchObject.Types.Properties BuildBatchProperties(TProps data continue; } + if (propertyInfo.PropertyType.IsArray) + { + switch (value) + { + case bool[] v: + props.BooleanArrayProperties.Add( + new V1.BooleanArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v }, + } + ); + break; + case int[] v: + props.IntArrayProperties.Add( + new V1.IntArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v.Select(Convert.ToInt64) }, + } + ); + break; + case long[] v: + props.IntArrayProperties.Add( + new V1.IntArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v }, + } + ); + break; + case double[] v: + props.NumberArrayProperties.Add( + new V1.NumberArrayProperties() + { + PropName = propertyInfo.Name, + ValuesBytes = v.ToByteString(), + } + ); + break; + case float[] v: + props.NumberArrayProperties.Add( + new V1.NumberArrayProperties() + { + PropName = propertyInfo.Name, + ValuesBytes = v.Select(Convert.ToDouble).ToByteString(), + } + ); + break; + case string[] v: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v }, + } + ); + break; + case Guid[] v: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v.Select(v => v.ToString()) }, + } + ); + break; + case DateTime[] v: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v.Select(v => v.ToUniversalTime().ToString("o")) }, + } + ); + break; + case DateTimeOffset[] v: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { v.Select(dto => dto.ToUniversalTime().ToString("o")) }, + } + ); + break; + + // Handle general IEnumerable (e.g., List, HashSet) + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable bools: + props.BooleanArrayProperties.Add( + new V1.BooleanArrayProperties() + { + PropName = propertyInfo.Name, + Values = { bools }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable ints: + props.IntArrayProperties.Add( + new V1.IntArrayProperties() + { + PropName = propertyInfo.Name, + Values = { ints.Select(Convert.ToInt64) }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable longs: + props.IntArrayProperties.Add( + new V1.IntArrayProperties() + { + PropName = propertyInfo.Name, + Values = { longs }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable doubles: + props.NumberArrayProperties.Add( + new V1.NumberArrayProperties() + { + PropName = propertyInfo.Name, + ValuesBytes = doubles.ToByteString(), + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable floats: + props.NumberArrayProperties.Add( + new V1.NumberArrayProperties() + { + PropName = propertyInfo.Name, + ValuesBytes = floats.Select(f => (double)f).ToByteString(), + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable strings: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { strings }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable guids: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = { guids.Select(g => g.ToString()) }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable dateTimes: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = + { + dateTimes.Select(dt => dt.ToUniversalTime().ToString("o")), + }, + } + ); + continue; + case System.Collections.IEnumerable enumerable + when enumerable is IEnumerable dateTimeOffsets: + props.TextArrayProperties.Add( + new V1.TextArrayProperties() + { + PropName = propertyInfo.Name, + Values = + { + dateTimeOffsets.Select(dto => + dto.ToUniversalTime().ToString("o") + ), + }, + } + ); + continue; + default: + throw new WeaviateException( + $"Unsupported array type '{value.GetType().GetElementType()?.Name ?? value.GetType().Name}' for property '{propertyInfo.Name}'. Check the documentation for supported array value types." + ); + } + continue; // Move to the next property after handling array + } + if (propertyInfo.PropertyType.IsNativeType()) { nonRefProps ??= new(); - nonRefProps.Fields.Add(propertyInfo.Name, ConvertToValue(value)); + nonRefProps.Fields.Add(propertyInfo.Name, ConvertToProtoValue(value)); } } @@ -340,6 +494,60 @@ private V1.BatchObject.Types.Properties BuildBatchProperties(TProps data return props; } + // Helper method to convert C# objects to protobuf Values + internal static Value ConvertToProtoValue(object obj) + { + return obj switch + { + null => Value.ForNull(), + bool b => Value.ForBool(b), + int i => Value.ForNumber(i), + long l => Value.ForNumber(l), + float f => Value.ForNumber(f), + double d => Value.ForNumber(d), + decimal dec => Value.ForNumber((double)dec), + string s => Value.ForString(s), + DateTime dt => Value.ForString(dt.ToUniversalTime().ToString("o")), + Guid uuid => Value.ForString(uuid.ToString()), + GeoCoordinate v => Value.ForStruct( + new Struct + { + Fields = + { + ["latitude"] = Value.ForNumber(v.Latitude), + ["longitude"] = Value.ForNumber(v.Longitude), + }, + } + ), + // Dictionary dict => Value.ForStruct(CreateStructFromDictionary(dict)), + // IEnumerable enumerable => CreateListValue(enumerable), + _ => throw new ArgumentException($"Unsupported type: {obj.GetType()}"), + }; + } +} + +public class DataClient +{ + private readonly CollectionClient _collectionClient; + private WeaviateClient _client => _collectionClient.Client; + private string _collectionName => _collectionClient.Name; + + internal DataClient(CollectionClient collectionClient) + { + _collectionClient = collectionClient; + } + + public static IDictionary[] MakeBeacons(params Guid[] guids) + { + return + [ + .. guids.Select(uuid => new Dictionary + { + { "beacon", $"weaviate://localhost/{uuid}" }, + }), + ]; + } + public async Task Insert( TData data, Guid? id = null, @@ -392,7 +600,7 @@ params BatchInsertRequest[] requests { Collection = _collectionName, Uuid = (r.ID ?? Guid.NewGuid()).ToString(), - Properties = BuildBatchProperties(r.Data), + Properties = ObjectHelper.BuildBatchProperties(r.Data), }; if (r.References?.Any() ?? false) diff --git a/src/Weaviate.Client/Extensions.cs b/src/Weaviate.Client/Extensions.cs index d26f3aa1..886916c2 100644 --- a/src/Weaviate.Client/Extensions.cs +++ b/src/Weaviate.Client/Extensions.cs @@ -240,7 +240,7 @@ internal static IEnumerable FromByteString(this Google.Protobuf.ByteString } } - internal static MemoryStream ToStream(this IEnumerable items) + internal static Stream ToStream(this IEnumerable items) where T : struct { var stream = new MemoryStream(); @@ -251,9 +251,15 @@ internal static MemoryStream ToStream(this IEnumerable items) { switch (item) { + case long v: + writer.Write(v); + break; case int v: writer.Write(v); break; + case double v: + writer.Write(v); + break; case float v: writer.Write(v); break; @@ -271,7 +277,6 @@ internal static Google.Protobuf.ByteString ToByteString(this IEnumerable i where T : struct { using var stream = items.ToStream(); - return Google.Protobuf.ByteString.FromStream(stream); } @@ -324,6 +329,7 @@ public static bool IsNativeType(this Type type) // Check for other common native types if ( underlyingType == typeof(Guid) + || underlyingType == typeof(GeoCoordinate) || underlyingType == typeof(TimeSpan) || underlyingType == typeof(DateTimeOffset) || underlyingType == typeof(DateTime) diff --git a/src/Weaviate.Client/QueryClient.cs b/src/Weaviate.Client/QueryClient.cs index 5ac6ff08..212f46de 100644 --- a/src/Weaviate.Client/QueryClient.cs +++ b/src/Weaviate.Client/QueryClient.cs @@ -33,18 +33,20 @@ public async Task List( ); } - public async Task FetchObjectByID( + public async Task FetchObjectByID( Guid id, IList? references = null, MetadataQuery? metadata = null ) { - return await _client.GrpcClient.FetchObjects( - _collectionName, - filter: Filter.WithID(id), - reference: references, - metadata: metadata - ); + return ( + await _client.GrpcClient.FetchObjects( + _collectionName, + filter: Filter.WithID(id), + reference: references, + metadata: metadata + ) + ).SingleOrDefault(); } public async Task FetchObjectsByIDs(