From ef3073378211dc060f68b1003a8314d74537b9c9 Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Fri, 30 May 2025 19:48:23 +0200 Subject: [PATCH 1/3] Connection Helpers --- README.md | 31 +++++-- src/Example/Program.cs | 115 +++++++++++--------------- src/Weaviate.Client/Helpers.cs | 60 ++++++++++++++ src/Weaviate.Client/Rest/Client.cs | 10 +-- src/Weaviate.Client/WeaviateClient.cs | 69 ++++++++++++---- src/Weaviate.Client/gRPC/Client.cs | 32 +++++-- 6 files changed, 210 insertions(+), 107 deletions(-) create mode 100644 src/Weaviate.Client/Helpers.cs diff --git a/README.md b/README.md index 5cd12c53..96bbd4bb 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ ## Features +### Helpers + +- [x] Connection helpers: Local, Cloud, Custom, FromEnvironment + ### Collections - [x] List collections. @@ -14,16 +18,29 @@ ### Objects - [x] Insert data. - - [ ] Add object with vector data. + - [X] Add object with named vector data. - [x] Delete data. -- [ ] Update data. -- [ ] Get object by ID. +- [X] Update data. +- [X] Get object by ID. - [x] Get objects. - [x] Get object metadata (vectors, schema, etc.) ### Search -- [ ] Query objects over gRPC. -- [ ] Perform a search with: - - Search mode: BM5, Hybrid, Near vector. - - Pagination: Limit, Offset. +- [X] Query objects over gRPC. +- [X] Perform a search with: + - Search mode: + - [X] BM5 + - [X] Hybrid + - [X] Near vector + - Pagination: + - [X] Limit + - [ ] Offset. +- Filters + - [X] Property + - [X] Property Length + - [X] Creation/Update Time + - [X] Single-Target References + - [X] Reference Count + - [ ] Multi-Target References + - [ ] Geo Coordinates diff --git a/src/Example/Program.cs b/src/Example/Program.cs index a77a1f3e..6dd58b2b 100644 --- a/src/Example/Program.cs +++ b/src/Example/Program.cs @@ -1,36 +1,13 @@ - -using System.Text.Json; -using Weaviate.Client; +using System.Text.Json; using Weaviate.Client.Models; -using CatDataWithVectors = (Example.Cat Data, float[] Vector); - namespace Example; class Program { - static private ClientConfiguration GetConfiguration() - { - var instanceUrl = Environment.GetEnvironmentVariable("WEAVIATE_CLUSTER_URL"); - if (string.IsNullOrEmpty(instanceUrl)) - { - throw new Exception("Required environment variable WEAVIATE_CLUSTER_URL is missing."); - } - - var apiKey = Environment.GetEnvironmentVariable("WEAVIATE_API_KEY"); - if (string.IsNullOrEmpty(apiKey)) - { - throw new Exception("Required environment variable WEAVIATE_API_KEY is missing."); - } - - return new ClientConfiguration - { - Host = new Uri(instanceUrl), - ApiKey = apiKey - }; - } + private record CatDataWithVectors(float[] Vector, Cat Data); - static async Task> GetCatsAsync(string filename) + static async Task> GetCatsAsync(string filename) { try { @@ -40,13 +17,19 @@ static async Task> GetCatsAsync(string filename) return []; // Return an empty list if the file doesn't exist } - using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true)) - { - // Deserialize directly from the stream for better performance, especially with large files - var data = await JsonSerializer.DeserializeAsync>(fs) ?? []; + using FileStream fs = new FileStream( + filename, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + useAsync: true + ); - return data; - } + // Deserialize directly from the stream for better performance, especially with large files + var data = await JsonSerializer.DeserializeAsync>(fs) ?? []; + + return data; } catch (JsonException ex) { @@ -62,9 +45,13 @@ static async Task> GetCatsAsync(string filename) static async Task Main() { - //var config = GetConfiguration(); + // Read 250 cats from JSON file and unmarshal into Cat class + var cats = await GetCatsAsync("cats.json"); + + // Use the C# client to store all cats with a cat class + Console.WriteLine("Cats to store: " + cats.Count); - var weaviate = new WeaviateClient(); + var weaviate = Weaviate.Client.Connect.Local(); var collection = weaviate.Collections.Use("Cat"); @@ -93,22 +80,25 @@ static async Task Main() { Vectorizer = new Dictionary { - { "none", new object { } } + { + "none", + new object { } + }, }, VectorIndexType = "hnsw", }; var VectorConfigs = new Dictionary { - { "default", vectorizerConfigNone } + { "default", vectorizerConfigNone }, }; var catCollection = new Collection() { Name = "Cat", Description = "Lots of Cats of multiple breeds", - Properties = [Property.Text("Name"), Property.Text("Color"), Property.Text("Breed"), Property.Int("Counter")], - VectorConfig = VectorConfigs + Properties = Property.FromType(), + VectorConfig = VectorConfigs, }; collection = await weaviate.Collections.Create(catCollection); @@ -118,17 +108,9 @@ static async Task Main() Console.WriteLine($"Collection: {c.Name}"); } - // // Read 250 cats from JSON file and unmarshal into Cat class - var cats = await GetCatsAsync("cats.json"); - - // Use the C# client to store all cats with a cat class - Console.WriteLine("Cats to store: " + cats.Count()); foreach (var cat in cats) { - var vectors = new NamedVectors() - { - { "default", cat.Vector } - }; + var vectors = new NamedVectors() { { "default", cat.Vector } }; var inserted = await collection.Data.Insert(cat.Data, vectors: vectors); } @@ -154,37 +136,40 @@ static async Task Main() if (firstObj.ID is Guid id2) { var fetched = await collection.Query.FetchObjectByID(id: id2); - Console.WriteLine("Cat retrieved via gRPC matches: " + ((fetched?.Objects.First().ID ?? Guid.Empty) == id2)); + Console.WriteLine( + "Cat retrieved via gRPC matches: " + + ((fetched?.Objects.First().ID ?? Guid.Empty) == id2) + ); } { var idList = retrieved - .Where(c => c.ID.HasValue) - .Take(10) - .Select(c => c.ID!.Value) - .ToHashSet(); + .Where(c => c.ID.HasValue) + .Take(10) + .Select(c => c.ID!.Value) + .ToHashSet(); var fetched = await collection.Query.FetchObjectsByIDs(idList); - Console.WriteLine($"Cats retrieved via gRPC matches:{Environment.NewLine} {JsonSerializer.Serialize(fetched.Objects, new JsonSerializerOptions { WriteIndented = true })}"); + Console.WriteLine( + $"Cats retrieved via gRPC matches:{Environment.NewLine} {JsonSerializer.Serialize(fetched.Objects, new JsonSerializerOptions { WriteIndented = true })}" + ); } - var queryNearVector = - await collection - .Query - .NearVector( - vector: [20f, 21f, 22f], - distance: 0.5f, - limit: 5, - fields: ["name", "breed", "color", "counter"], - metadata: MetadataOptions.Score | MetadataOptions.Distance - ); + var queryNearVector = await collection.Query.NearVector( + vector: [20f, 21f, 22f], + distance: 0.5f, + limit: 5, + fields: ["name", "breed", "color", "counter"], + metadata: MetadataOptions.Score | MetadataOptions.Distance + ); foreach (var cat in queryNearVector.Objects) { - Console.WriteLine(JsonSerializer.Serialize(cat, new JsonSerializerOptions { WriteIndented = true })); + Console.WriteLine( + JsonSerializer.Serialize(cat, new JsonSerializerOptions { WriteIndented = true }) + ); } - // Cursor API // var objects = collection.Iterator(); // var sum = await objects.SumAsync(c => c.Counter); diff --git a/src/Weaviate.Client/Helpers.cs b/src/Weaviate.Client/Helpers.cs new file mode 100644 index 00000000..fbcb0fc6 --- /dev/null +++ b/src/Weaviate.Client/Helpers.cs @@ -0,0 +1,60 @@ +namespace Weaviate.Client; + +public static class Connect +{ + public static WeaviateClient Local( + ushort restPort = 8080, + ushort grpcPort = 50051, + string? apiKey = null, + bool useSsl = false + ) => new(new ClientConfiguration("localhost", "localhost", apiKey, restPort, grpcPort, useSsl)); + + public static WeaviateClient Cloud(string cloudUrl, string? apiKey = null) => + new(new ClientConfiguration(cloudUrl, $"grpc-{cloudUrl}", apiKey)); + + public static WeaviateClient FromEnvironment(string prefix = "WEAVIATE_") + { + var restEndpoint = Environment.GetEnvironmentVariable($"{prefix}REST_ENDPOINT"); + var grpcEndpoint = Environment.GetEnvironmentVariable($"{prefix}GRPC_ENDPOINT"); + var restPort = Environment.GetEnvironmentVariable($"{prefix}REST_PORT") ?? "8080"; + var grpcPort = Environment.GetEnvironmentVariable($"{prefix}GRPC_PORT") ?? "50051"; + var useSsl = Environment.GetEnvironmentVariable($"{prefix}USE_SSL")?.ToLower() == "true"; + var apiKey = Environment.GetEnvironmentVariable($"{prefix}API_KEY"); + + if (restEndpoint is null && grpcEndpoint is null) + { + throw new InvalidOperationException("No REST or GRPC endpoint provided."); + } + else if (restEndpoint is not null && grpcEndpoint is null) + { + grpcEndpoint = restEndpoint; + } + else if (restEndpoint is null && grpcEndpoint is not null) + { + restEndpoint = grpcEndpoint; + } + + return Custom(restEndpoint!, grpcEndpoint!, restPort, grpcPort, useSsl, apiKey); + } + + private static WeaviateClient Custom( + string restEndpoint, + string grpcEndpoint, + string restPort, + string grpcPort, + bool useSsl, + string? apiKey + ) + { + return new( + new ClientConfiguration( + restEndpoint, + grpcEndpoint, + apiKey, + Convert.ToUInt16(restPort), + Convert.ToUInt16(grpcPort), + useSsl + ) + ); + } +} diff --git a/src/Weaviate.Client/Rest/Client.cs b/src/Weaviate.Client/Rest/Client.cs index c64f99b3..b52efc6b 100644 --- a/src/Weaviate.Client/Rest/Client.cs +++ b/src/Weaviate.Client/Rest/Client.cs @@ -88,7 +88,7 @@ public class WeaviateRestClient : IDisposable private readonly bool _ownershipClient; private readonly HttpClient _httpClient; - internal WeaviateRestClient(WeaviateClient client, HttpClient? httpClient = null) + internal WeaviateRestClient(Uri restUri, HttpClient? httpClient = null) { if (httpClient is null) { @@ -97,13 +97,7 @@ internal WeaviateRestClient(WeaviateClient client, HttpClient? httpClient = null } _httpClient = httpClient; - - var ub = new UriBuilder(client.Configuration.Host); - - ub.Port = client.Configuration.RestPort; - ub.Path = "v1/"; - - _httpClient.BaseAddress = ub.Uri; + _httpClient.BaseAddress = restUri; } public void Dispose() diff --git a/src/Weaviate.Client/WeaviateClient.cs b/src/Weaviate.Client/WeaviateClient.cs index 48e3ef01..46904d47 100644 --- a/src/Weaviate.Client/WeaviateClient.cs +++ b/src/Weaviate.Client/WeaviateClient.cs @@ -6,32 +6,54 @@ namespace Weaviate.Client; -public record ClientConfiguration +public record ClientConfiguration( + string RestAddress = "localhost", + string GrpcAddress = "localhost", + string? ApiKey = null, + ushort RestPort = 8080, + ushort GrpcPort = 50051, + bool UseSsl = false +) { - public required Uri Host; - public ushort RestPort = 8080; - public ushort GrpcPort = 50051; - public required string ApiKey; -} + public virtual Uri RestUri => + new UriBuilder() + { + Host = RestAddress, + Scheme = UseSsl ? "https" : "http", + Port = RestPort, + Path = "v1/", + }.Uri; + + public virtual Uri GrpcUri => + new UriBuilder() + { + Host = GrpcAddress, + Scheme = UseSsl ? "https" : "http", + Port = GrpcPort, + Path = "", + }.Uri; +}; public class WeaviateClient : IDisposable { - private static readonly Lazy _defaultOptions = new Lazy(() => new() - { - Host = new Uri("http://localhost"), - ApiKey = "", - }); + private static readonly Lazy _defaultOptions = new(() => + new() + { + ApiKey = null, + RestPort = 8080, + GrpcPort = 50051, + } + ); public static ClientConfiguration DefaultOptions => _defaultOptions.Value; private bool _isDisposed = false; - private readonly WeaviateGrpcClient _grpcClient; - private readonly WeaviateRestClient _restClient; [NotNull] - internal WeaviateRestClient RestClient => _restClient; + internal WeaviateRestClient RestClient { get; init; } + [NotNull] - internal WeaviateGrpcClient GrpcClient => _grpcClient; + internal WeaviateGrpcClient GrpcClient { get; init; } public ClientConfiguration Configuration { get; } @@ -41,8 +63,19 @@ public WeaviateClient(ClientConfiguration? configuration = null, HttpClient? htt { Configuration = configuration ?? DefaultOptions; - _restClient = new WeaviateRestClient(this, httpClient); - _grpcClient = new WeaviateGrpcClient(this); + httpClient ??= new HttpClient(); + + if (!string.IsNullOrEmpty(Configuration.ApiKey)) + { + httpClient.DefaultRequestHeaders.Authorization = + new System.Net.Http.Headers.AuthenticationHeaderValue( + "Bearer", + Configuration.ApiKey + ); + } + + RestClient = new WeaviateRestClient(Configuration.RestUri, httpClient); + GrpcClient = new WeaviateGrpcClient(Configuration.GrpcUri, Configuration.ApiKey); Collections = new CollectionsClient(this); } @@ -52,7 +85,7 @@ public void Dispose() if (_isDisposed) return; - _grpcClient?.Dispose(); + GrpcClient?.Dispose(); RestClient?.Dispose(); _isDisposed = true; diff --git a/src/Weaviate.Client/gRPC/Client.cs b/src/Weaviate.Client/gRPC/Client.cs index d412823a..5b50d016 100644 --- a/src/Weaviate.Client/gRPC/Client.cs +++ b/src/Weaviate.Client/gRPC/Client.cs @@ -1,5 +1,6 @@ using System.Collections; using System.Dynamic; +using Grpc.Core; using Grpc.Net.Client; using Weaviate.V1; @@ -7,23 +8,36 @@ namespace Weaviate.Client.Grpc; public partial class WeaviateGrpcClient : IDisposable { - private readonly WeaviateClient _client; private readonly GrpcChannel _channel; private readonly V1.Weaviate.WeaviateClient _grpcClient; - public WeaviateGrpcClient(WeaviateClient client) + AsyncAuthInterceptor _AuthInterceptorFactory(string apiKey) { - _client = client; + return ( + async (context, metadata) => + { + metadata.Add("Authorization", $"Bearer {apiKey}"); + await Task.CompletedTask; + } + ); + } - var ub = new UriBuilder(client.Configuration.Host); + public WeaviateGrpcClient(Uri grpcUri, string? apiKey = null) + { + var options = new GrpcChannelOptions(); - ub.Port = client.Configuration.GrpcPort; + if (apiKey != null) + { + var credentials = CallCredentials.FromInterceptor(_AuthInterceptorFactory(apiKey)); + options.Credentials = ChannelCredentials.Create(new SslCredentials(), credentials); + } + ; - _channel = GrpcChannel.ForAddress(ub.Uri); + _channel = GrpcChannel.ForAddress(grpcUri, options); _grpcClient = new V1.Weaviate.WeaviateClient(_channel); } - private static IList buildListFromListValue(ListValue list) + private static IList MakeListValue(ListValue list) { switch (list.KindCase) { @@ -77,7 +91,7 @@ private static ExpandoObject MakeNonRefs(Properties result) eo[r.Key] = MakeNonRefs(r.Value.ObjectValue) ?? new object { }; break; case Value.KindOneofCase.ListValue: - eo[r.Key] = buildListFromListValue(r.Value.ListValue); + eo[r.Key] = MakeListValue(r.Value.ListValue); break; case Value.KindOneofCase.DateValue: eo[r.Key] = r.Value.DateValue; // TODO Parse date here? @@ -110,4 +124,4 @@ public void Dispose() { _channel.Dispose(); } -} \ No newline at end of file +} From a643109f6fe76e970ec398d94c9225a87fa6724b Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Fri, 30 May 2025 19:51:17 +0200 Subject: [PATCH 2/3] Fix cloud ports --- src/Weaviate.Client/Helpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Weaviate.Client/Helpers.cs b/src/Weaviate.Client/Helpers.cs index fbcb0fc6..cd986952 100644 --- a/src/Weaviate.Client/Helpers.cs +++ b/src/Weaviate.Client/Helpers.cs @@ -10,7 +10,7 @@ public static WeaviateClient Local( ) => new(new ClientConfiguration("localhost", "localhost", apiKey, restPort, grpcPort, useSsl)); public static WeaviateClient Cloud(string cloudUrl, string? apiKey = null) => - new(new ClientConfiguration(cloudUrl, $"grpc-{cloudUrl}", apiKey)); + new(new ClientConfiguration(cloudUrl, $"grpc-{cloudUrl}", apiKey, 443, 443, true)); public static WeaviateClient FromEnvironment(string prefix = "WEAVIATE_") { From 05359ebd985635c6aa47b0ecff67cd1b8b1e8da2 Mon Sep 17 00:00:00 2001 From: Michelangelo Partipilo Date: Fri, 30 May 2025 20:18:55 +0200 Subject: [PATCH 3/3] Cleanup and Tests --- .../Integration/Connection.cs | 29 +++++++++++++++++++ src/Weaviate.Client/Helpers.cs | 24 ++++++++++----- src/Weaviate.Client/WeaviateClient.cs | 12 ++++---- 3 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 src/Weaviate.Client.Tests/Integration/Connection.cs diff --git a/src/Weaviate.Client.Tests/Integration/Connection.cs b/src/Weaviate.Client.Tests/Integration/Connection.cs new file mode 100644 index 00000000..330691a0 --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/Connection.cs @@ -0,0 +1,29 @@ +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + [Fact] + public async Task ConnectToLocal() + { + var client = Connect.Local(); + + var ex = await Record.ExceptionAsync(() => + client.Collections.List().ToListAsync(TestContext.Current.CancellationToken).AsTask() + ); + Assert.Null(ex); + } + + [Fact] + public async Task ConnectToCloud() + { + var WCS_HOST = "piblpmmdsiknacjnm1ltla.c1.europe-west3.gcp.weaviate.cloud"; + var WCS_CREDS = "cy4ua772mBlMdfw3YnclqAWzFhQt0RLIN0sl"; + + var client = Connect.Cloud(WCS_HOST, WCS_CREDS); + + var ex = await Record.ExceptionAsync(() => + client.Collections.List().ToListAsync(TestContext.Current.CancellationToken).AsTask() + ); + Assert.Null(ex); + } +} diff --git a/src/Weaviate.Client/Helpers.cs b/src/Weaviate.Client/Helpers.cs index cd986952..4e203145 100644 --- a/src/Weaviate.Client/Helpers.cs +++ b/src/Weaviate.Client/Helpers.cs @@ -2,15 +2,25 @@ namespace Weaviate.Client; public static class Connect { + public static ClientConfiguration LocalConfig( + ushort restPort, + ushort grpcPort, + bool useSsl, + string? apiKey + ) => new("localhost", "localhost", restPort, grpcPort, useSsl, apiKey); + public static WeaviateClient Local( ushort restPort = 8080, ushort grpcPort = 50051, - string? apiKey = null, - bool useSsl = false - ) => new(new ClientConfiguration("localhost", "localhost", apiKey, restPort, grpcPort, useSsl)); + bool useSsl = false, + string? apiKey = null + ) => LocalConfig(restPort, grpcPort, useSsl, apiKey).Client(); + + public static ClientConfiguration CloudConfig(string restEndpoint, string? apiKey = null) => + new ClientConfiguration(restEndpoint, $"grpc-{restEndpoint}", 443, 443, true, apiKey); - public static WeaviateClient Cloud(string cloudUrl, string? apiKey = null) => - new(new ClientConfiguration(cloudUrl, $"grpc-{cloudUrl}", apiKey, 443, 443, true)); + public static WeaviateClient Cloud(string restEndpoint, string? apiKey = null) => + CloudConfig(restEndpoint, apiKey).Client(); public static WeaviateClient FromEnvironment(string prefix = "WEAVIATE_") { @@ -50,10 +60,10 @@ private static WeaviateClient Custom( new ClientConfiguration( restEndpoint, grpcEndpoint, - apiKey, Convert.ToUInt16(restPort), Convert.ToUInt16(grpcPort), - useSsl + useSsl, + apiKey ) ); } diff --git a/src/Weaviate.Client/WeaviateClient.cs b/src/Weaviate.Client/WeaviateClient.cs index 46904d47..fdd16e4c 100644 --- a/src/Weaviate.Client/WeaviateClient.cs +++ b/src/Weaviate.Client/WeaviateClient.cs @@ -6,16 +6,16 @@ namespace Weaviate.Client; -public record ClientConfiguration( +public sealed record ClientConfiguration( string RestAddress = "localhost", string GrpcAddress = "localhost", - string? ApiKey = null, ushort RestPort = 8080, ushort GrpcPort = 50051, - bool UseSsl = false + bool UseSsl = false, + string? ApiKey = null ) { - public virtual Uri RestUri => + public Uri RestUri => new UriBuilder() { Host = RestAddress, @@ -24,7 +24,7 @@ public record ClientConfiguration( Path = "v1/", }.Uri; - public virtual Uri GrpcUri => + public Uri GrpcUri => new UriBuilder() { Host = GrpcAddress, @@ -32,6 +32,8 @@ public record ClientConfiguration( Port = GrpcPort, Path = "", }.Uri; + + public WeaviateClient Client() => new(this); }; public class WeaviateClient : IDisposable