diff --git a/src/Example/Program.cs b/src/Example/Program.cs index 4b0be5ea..343fd749 100644 --- a/src/Example/Program.cs +++ b/src/Example/Program.cs @@ -172,11 +172,11 @@ static async Task Main() collection .Query .NearVector( - vector: new float[] { 20f, 21f, 22f }, + vector: [20f, 21f, 22f], distance: 0.5f, limit: 5, fields: ["name", "breed", "color", "counter"], - metadata: ["score", "distance"] + metadata: MetadataOptions.Score | MetadataOptions.Distance ); await foreach (var cat in queryNearVector) diff --git a/src/Weaviate.Client.Tests/Integration/Data.cs b/src/Weaviate.Client.Tests/Integration/Data.cs new file mode 100644 index 00000000..4d825a53 --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/Data.cs @@ -0,0 +1,53 @@ +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + [Fact] + public async Task CollectionCreation() + { + // Arrange + + // Act + var collectionClient = await CollectionFactory("", "Test collection description", [ + Property.Text("Name") + ]); + + // Assert + var collection = await _weaviate.Collections.Use(collectionClient.Name).Get(); + Assert.NotNull(collection); + Assert.Equal("CollectionCreation", collection.Name); + Assert.Equal("Test collection description", collection.Description); + } + + [Fact] + public async Task ObjectCreation() + { + // Arrange + var collectionClient = await CollectionFactory("", "Test collection description", [ + Property.Text("Name") + ]); + + // Act + var id = Guid.NewGuid(); + var obj = await collectionClient.Data.Insert(new WeaviateObject() + { + Data = new TestData { Name = "TestObject" }, + ID = id, + }); + + // Assert + + // Assert object exists + var retrieved = await collectionClient.Query.FetchObjectByID(id); + Assert.NotNull(retrieved); + Assert.Equal(id, retrieved.ID); + Assert.Equal("TestObject", retrieved.Data?.Name); + + // Delete after usage + await collectionClient.Data.Delete(id); + retrieved = await collectionClient.Query.FetchObjectByID(id); + Assert.Null(retrieved); + } +} \ No newline at end of file diff --git a/src/Weaviate.Client.Tests/Integration/NearText.cs b/src/Weaviate.Client.Tests/Integration/NearText.cs new file mode 100644 index 00000000..b65907f6 --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/NearText.cs @@ -0,0 +1,119 @@ + +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + [Fact] + public async Task NearTextSearch() + { + // Arrange + var collectionClient = await CollectionFactory("", "Test collection description", [ + Property.Text("value") + ], vectorConfig: new Dictionary + { + { + "default", new VectorConfig + { + Vectorizer = new Dictionary { + { + "text2vec-contextionary", new { + vectorizeClassName = false + } + } + }, + VectorIndexType = "hnsw" + } + } + }); + + string[] values = ["Apple", "Mountain climbing", "apple cake", "cake"]; + var tasks = values.Select(s => new TestDataValue { Value = s }) + .Select(DataFactory) + .Select(d => collectionClient.Data.Insert(d)); + Guid[] guids = await Task.WhenAll(tasks); + var concepts = "hiking"; + + // Act + var retriever = collectionClient.Query.NearText( + "cake", + moveTo: new Move(1.0f, objects: guids[0]), + moveAway: new Move(0.5f, concepts: concepts), + fields: ["value"], + metadata: new MetadataQuery(Vectors: new HashSet(["default"])) + ); + var retrieved = await retriever.ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(retrieved); + Assert.Equal(4, retrieved.Count()); + + Assert.Equal(retrieved[0].ID, guids[2]); + Assert.Contains("default", retrieved[0].Vectors.Keys); + Assert.Equal("apple cake", retrieved[0].Data?.Value); + } + + [Fact] + public async Task NearTextGroupBySearch() + { + // Arrange + CollectionClient? collectionClient = await CollectionFactory("", "Test collection description", [ + Property.Text("value") + ], vectorConfig: new Dictionary + { + { + "default", new VectorConfig + { + Vectorizer = new Dictionary { + { + "text2vec-contextionary", new { + vectorizeClassName = false + } + } + }, + VectorIndexType = "hnsw" + } + } + }); + + string[] values = ["Apple", "Mountain climbing", "apple cake", "cake"]; + var tasks = values.Select(s => new { Value = s }) + .Select(DataFactory) + .Select(d => collectionClient.Data.Insert(d)); + Guid[] guids = await Task.WhenAll(tasks); + + // Act + var retrieved = await collectionClient.Query.NearText( + "cake", + new GroupByConstraint + { + PropertyName = "value", + NumberOfGroups = 2, + ObjectsPerGroup = 100, + }, + metadata: new MetadataQuery(Vectors: ["default"]) + ); + + // Assert + Assert.NotNull(retrieved.Objects); + Assert.NotNull(retrieved.Groups); + + var retrievedObjects = retrieved.Objects.ToArray(); + + Assert.Equal(2, retrieved.Objects.Count()); + Assert.Equal(2, retrieved.Groups.Count()); + + var obj = await collectionClient.Query.FetchObjectByID(guids[3], metadata: new MetadataQuery(Vectors: ["default"])); + Assert.NotNull(obj); + Assert.Equal(guids[3], obj.ID); + Assert.Contains("default", obj.Vectors.Keys); + + Assert.Equal(guids[3], retrievedObjects[0].ID); + Assert.Contains("default", retrievedObjects[0].Vectors.Keys); + Assert.Equal("cake", retrievedObjects[0].BelongsToGroup); + Assert.Equal(guids[2], retrievedObjects[1].ID); + Assert.Contains("default", retrievedObjects[1].Vectors.Keys); + Assert.Equal("apple cake", retrievedObjects[1].BelongsToGroup); + } +} \ No newline at end of file diff --git a/src/Weaviate.Client.Tests/Integration/NearVector.cs b/src/Weaviate.Client.Tests/Integration/NearVector.cs new file mode 100644 index 00000000..73de650a --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/NearVector.cs @@ -0,0 +1,54 @@ +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + + [Fact] + public async Task NearVectorSearch() + { + // Arrange + var collectionClient = await CollectionFactory("", "Test collection description", [ + Property.Text("Name") + ]); + + // Act + await collectionClient.Data.Insert(new WeaviateObject() + { + Data = new TestData { Name = "TestObject1" }, + Vectors = new Dictionary> + { + { "default", new float[] { 0.1f, 0.2f, 0.3f } } + } + }); + + await collectionClient.Data.Insert(new WeaviateObject() + { + Data = new TestData { Name = "TestObject2" }, + Vectors = new Dictionary> + { + { "default", new float[] { 0.3f, 0.4f, 0.5f } } + } + }); + + await collectionClient.Data.Insert(new WeaviateObject() + { + Data = new TestData { Name = "TestObject3" }, + Vectors = new Dictionary> + { + { "default", new float[] { 0.5f, 0.6f, 0.7f } } + } + }); + + // Assert + var retrieved = collectionClient.Query.NearVector(new float[] { 0.1f, 0.2f, 0.3f }); + Assert.NotNull(retrieved); + + await foreach (var obj in retrieved) + { + Assert.Equal("TestObject1", obj.Data!.Name); + break; + } + } +} \ No newline at end of file diff --git a/src/Weaviate.Client.Tests/Integration/SingleTargetRef.cs b/src/Weaviate.Client.Tests/Integration/SingleTargetRef.cs new file mode 100644 index 00000000..eb7d3d4e --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/SingleTargetRef.cs @@ -0,0 +1,330 @@ + +using System.Text.Json; +using Weaviate.Client.Models; + +namespace Weaviate.Client.Tests.Integration; + +public partial class BasicTests +{ + readonly Guid TO_UUID = new Guid("8ad0d33c-8db1-4437-87f3-72161ca2a51a"); + readonly Guid TO_UUID2 = new Guid("577887c1-4c6b-5594-aa62-f0c17883d9cf"); + + [Fact] + public async Task SingleTargetReference() + { + // Arrange + + var cA = await CollectionFactory( + "A", + "Collection A", + [ + Property.Text("Name") + ]); + + var uuid_A1 = await cA.Data.Insert(DataFactory(new TestData() { Name = "A1" })); + var uuid_A2 = await cA.Data.Insert(DataFactory(new TestData() { Name = "A2" })); + + var cB = await CollectionFactory( + name: "B", + description: "Collection B", + properties: [Property.Text("Name")], + references: [Property.Reference("a", cA.Name)]); + + var uuid_B = await cB.Data.Insert( + DataFactory(new TestData { Name = "B" }), + new Dictionary { + { "a", uuid_A1 } + }); + + await cB.Data.ReferenceAdd(from: uuid_B, fromProperty: "a", to: uuid_A2); + + var cC = await CollectionFactory( + "C", + "Collection C", + [ + Property.Text("Name") + ], + references: [Property.Reference("b", cB.Name)]); + + var uuid_C = await cC.Data.Insert(DataFactory(new TestData { Name = "find me" }), + new Dictionary { + { "b", uuid_B } + }); + + // Act + var aObjs = (await cA.Query.BM25(query: "A1", ["name"])).ToList(); + + var bObjs = (await cB.Query.BM25( + query: "B", + references: + [new QueryReference( + linkOn: "a", + fields: ["name"] + )] + )).ToList(); + + var cObjs = (await cC.Query.BM25( + query: "find", + fields: ["name"], + references: + [new QueryReference( + linkOn: "b", + fields: ["name"], + metadata: MetadataOptions.LastUpdateTime, + references: [new QueryReference( + linkOn: "a", + fields: ["name"] + )] + )] + )).ToList(); + + // Assert + Assert.Equal(cA.Name, aObjs[0].CollectionName); + Assert.NotNull(aObjs[0].Data?.Name); + Assert.Equal("A1", aObjs[0].Data?.Name); + + Assert.Equal(cA.Name, bObjs[0].References["a"][0].CollectionName); + Assert.Equal("A1", bObjs[0].References["a"][0].Properties["name"]); + Assert.Equal(uuid_A1, bObjs[0].References["a"][0].ID); + Assert.Equal(cA.Name, bObjs[0].References["a"][1].CollectionName); + Assert.Equal("A2", bObjs[0].References["a"][1].Properties["name"]); + Assert.Equal(uuid_A2, bObjs[0].References["a"][1].ID); + + + Assert.Equal(cC.Name, cObjs[0].CollectionName); + Assert.Equal("find me", cObjs[0].Properties["name"]); + Assert.Equal(uuid_C, cObjs[0].ID); + Assert.Equal(cB.Name, cObjs[0].References["b"][0].CollectionName); + Assert.Equal("B", cObjs[0].References["b"][0].Properties["name"]); + Assert.NotNull(cObjs[0].References["b"][0].Metadata.LastUpdateTime); + Assert.Equal(cA.Name, cObjs[0].References["b"][0].References["a"][0].CollectionName); + Assert.Equal("A1", cObjs[0].References["b"][0].References["a"][0].Properties["name"]); + Assert.Equal(cA.Name, cObjs[0].References["b"][0].References["a"][1].CollectionName); + Assert.Equal("A2", cObjs[0].References["b"][0].References["a"][1].Properties["name"]); + } + + [Fact] + public async Task SingleTargetReference_MovieReviews() + { + + // Arrange + var movies = await CollectionFactory( + name: "Movie", + "Movies", + properties: [ + Property.Text("title"), + Property.Text("overview"), + Property.Int("movie_id"), + Property.Date("release_date"), + Property.Int("vote_count"), + ]); + + var reviews = await CollectionFactory( + name: "Review", + "Movie Reviews", + properties: [ + Property.Text("author_username"), + Property.Text("content"), + Property.Int("rating"), + Property.Int("review_id"), + Property.Int("movie_id"), + Property.Reference("forMovie", targetCollection: "Movie"), + ], + vectorConfig: new Dictionary + { + { + "default", new VectorConfig + { + Vectorizer = new Dictionary { + { + "text2vec-contextionary", new { + vectorizeClassName = false + } + } + }, + VectorIndexType = "hnsw" + } + } + } + ); + + var moviesData = new[] { + new { + movie_id = 162, + title = "Edward Scissorhands", + overview = "A small suburban town receives a visit from a castaway unfinished science experiment named Edward.", + release_date = DateTime.Parse("1990-12-07").ToUniversalTime(), + vote_count = 12308 + }, + new { + movie_id = 769, + title = "GoodFellas", + overview = "The true story of Henry Hill, a half-Irish, half-Sicilian Brooklyn kid who is adopted by neighbourhood gangsters at an early age and climbs the ranks of a Mafia family under the guidance of Jimmy Conway.", + release_date = DateTime.Parse("1990-09-12").ToUniversalTime(), + vote_count = 12109 + }, + new { + movie_id = 771, + title = "Home Alone", + overview = "Eight-year-old Kevin McCallister makes the most of the situation after his family unwittingly leaves him behind when they go on Christmas vacation. But when a pair of bungling burglars set their sights on Kevin's house, the plucky kid stands ready to defend his territory. By planting booby traps galore, adorably mischievous Kevin stands his ground as his frantic mother attempts to race home before Christmas Day.", + release_date = DateTime.Parse("1990-11-16").ToUniversalTime(), + vote_count = 10601 + } + }; + + var movieIds = new Dictionary(); + foreach (var m in moviesData) + { + var uuid = await movies.Data.Insert(DataFactory(m)); + movieIds.Add(m.movie_id, uuid); + } + + var reviewsData = new List + { + new { + author_username = "kineticandroid", + content = @"Take the story of Frankenstein's monster, remove the hateful creator, and replace the little girl's flowers with a brightly pastel Reagan-era suburb. Though not my personal favorite Tim Burton film, I feel like this one best encapsulates his style and story interests.", + rating = (double?)null, + movie_id = 162, + review_id = 162 + }, + new { + author_username = "r96sk", + content = @"Very enjoyable. + +It's funny the way we picture things in our minds. I had heard of 'Edward Scissorhands' but actually knew very little about it, typified by the fact I was expecting this to be very dark - probably just based on the seeing the cover here and there. It's much sillier than expected, but in a positive way. + +I do kinda end up wishing they went down a more dark/creative route, instead of relying on the novelty of having scissors as hands; though, to be fair, they do touch on the deeper side a bit. With that said, I did get a good amount of entertainment seeing this plot unfold. It's weird and wonderful. + +Johnny Depp is a great actor and is very good here, mainly via his facial expressions and body language. It's cool to see Winona Ryder involved, someone I've thoroughly enjoyed in more recent times in 'Stranger Things'. Alan Arkin and Anthony Michael Hall also appear. + +The film looks neat, as I've come to expect from Tim Burton. It has the obvious touch of Bo Welch to it, with the neighbourhood looking not too dissimilar to what Welch would create for 2003's 'The Cat in the Hat' - which I, truly, enjoyed. + +Undoubtedly worth a watch.", + rating = (double?)8.0, + movie_id = 162, + review_id = 162 + }, + new { + author_username = "SoSmooth1982", + content = @"Love this movie. It's like a non evil Freddy Kruger. The ending could have been better though.", + rating = (double?)8.0, + movie_id = 162, + review_id = 162 + }, + new { + author_username = "Geronimo1967", + content = @"Vincent Price has spent his life working on a labour of love - a ""son"", an artificially constructed person that lacks only hands - for which he temporarily has two pairs of scissors. Sadly, the creator dies before he can rectify this and so young ""Edward"" (Johnny Depp) is left alone in his lofty castle. Alone, that is until a kindly Dianne Wiest (""Peg"") takes him under her wing, introduces him to her many friends - including an on-form Winona Ryder (""Kim"") - and they all discover he has a remarkable ability for topiary (and hairdressing!). Soon he is all the rage, the talk of the town - but always the misfit, and of course when a mishap - in this case a robbery for which he is framed - occurs, his fickle friends turn on him readily. It's a touching tale of innocence and humanity; Depp plays his role skilfully and with delicacy and humour, and the last half hour is quite a damning indictment of thoughtlessness and selfishness that still resonates today. Like many ""fairy"" tales, it has it's root in decent morals and Tim Burton is ahead of the game in delivering a nuanced and enjoyable modern day parable that makes you laugh, smile and wince with shame in equal measure.", + rating = (double?)7.0, + movie_id = 162, + review_id = 162 + }, + new { + author_username = "John Chard", + content = @"In a world that's powered by violence, on the streets where the violent have power, a new generation carries on an old tradition. + +Martin Scorsese’s Goodfellas is without question one of the finest gangster movies ever made, a benchmark even. It’s that rare occasion for a genre film of this type where everything artistically comes together as one. Direction, script, editing, photography, driving soundtrack and crucially an ensemble cast firing on all cylinders. It’s grade “A” film making that marked a return to form for Scorsese whilst simultaneously showing the director at the summit of his directing abilities. + +The story itself, based on Nicholas Pileggi’s non-fiction book Wiseguy, pulls absolutely no punches in its stark realisation of the Mafia lifestyle. It’s often brutal, yet funny, unflinching yet stylish, but ultimately from first frame to last it holds the attention, toying with all the human emotions during the journey, tingling the senses of those who were by 1990 fed up of popcorn movie fodder. + +It’s not romanticism here, if anything it’s a debunking of the Mafia myth, but even as the blood flows and the dialogue crackles with electricity, it always remains icy cool, brought to us by a man who had is eyes and ears open while growing up in Queens, New York in the 40s and 50s. Eccellente! 9/10", + rating = (double?)9.0, + movie_id = 769, + review_id = 769 + }, + new { + author_username = "Ahmetaslan27", + content = @"Martin Scorsese (director) always loves details in crime films, but he is not primarily interested in the crime itself. That is why his films are always produced with details that you may see as unimportant to you, especially if you want to see the movie for the purpose of seeing scenes of theft, murder, and so on, but you see the opposite. Somewhat other details are visible on the scene mostly + +The film talks about liberation, stereotypes, and entering a new world for humanity. It was Ray Liotta (Henry). He wanted, as I said, to break free from stereotypes and enter the world of gangs. + +Martin Scorsese (the director) filmed this unfamiliar life and directed it in the form of a film similar to documentaries because he filmed it as if it were a real, realistic life. That is why the presence of Voice Over was important in order to give you the feeling that there is a person sitting next to you telling you the story while whispering in your ear as it happens in the movies documentaries.", + rating = (double?)7.0, + movie_id = 769, + review_id = 769 + }, + new { + author_username = "Geronimo1967", + content = @"Ray Liotta is superb here as ""Henry Hill"", a man whom ever since he was young has been captivated by the mob. He starts off as a runner and before too long has ingratiated himself with the local fraternity lead by ""Paulie"" (Paul Sorvino) and is best mates with fellow hoods, the enigmatic and devious ""Jimmy"" (Robert De Niro) and the excellently vile ""Tommy"" (Joe Pesci). They put together an audacious robbery at JFK and are soon the talk of the town, but the latter in the trio is a bit of a live-wire and when he goes just a bit too far one night, the three of them find that their really quite idyllic lives of extortion and larceny start to go awry - and it's their own who are on their tracks. Scorsese takes him time with this story: the development of the characters - their personalities, trust, inter-reliance, sometimes divided, fractured, loyalties and ruthlessness and are built up in a thoroughly convincing fashion. We can, ourselves, see the obvious attractions for the young ""Henry"" of a life so very far removed from his working class Irish-Italian background - the wine, the women, the thrills; it's tantalising! If anything let's it down it's the last half hour; it's just a little too predictable and having spent so long building up the characters, we seem to be in just a bit too much of a rush; but that is a nit-pick. It's not the ""Godfather"" but it is not far short.", + rating = (double?)7.0, + movie_id = 769, + review_id = 769 + }, + new { + author_username = "Ruuz", + content = @"Doesn't really work if you actually spend the time to bother thinking about it, but so long as you don't _Home Alone_ is a pretty good time. There's really no likeable character, and it's honestly pretty mean spirited, but sometimes that's what you might need to defrag over Christmas. + +_Final rating:★★★ - I liked it. Would personally recommend you give it a go._", + rating = (double?)6.0, + movie_id = 771, + review_id = 771 + }, + new { + author_username = "SoSmooth1982", + content = @"Love this movie. I was 8 when this came out. I remember being so jealous of Kevin, because I wished I could be home alone like that to do whatever I wanted.", + rating = (double?)10.0, + movie_id = 771, + review_id = 771 + }, + new { + author_username = "Geronimo1967", + content = @"It has taken me 30 years to sit down and watch this film and I'm quite glad I finally did. I usually loathe kids movies, and the trails at the time always put me off - but Macauley Culkin is really quite a charmer in this tale of a youngster who is accidentally left at home at Christmas by his family. They have jetted off to Paris leaving him alone facing the unwanted attentions of two would-be burglars (Joe Pesci & Daniel Stern). Initially a bit unsettled, he is soon is his stride using just about every gadget (and critter) in their large family home to make sure he thwarts their thieving intentions. It's really all about the kid - and this one delivers well. The slapstick elements of the plot are designed to raise a smile, never to maim - even if having your head set on fire by a blow torch, or being walloped in the face by an hot iron might do longer term damage than happens here. That's the fun of it, for fun it is - it's a modern day Laurel & Hardy style story with an ending that's never in doubt. It does have a slightly more serious purpose, highlighting loneliness - not just for ""Kevin"" but his elderly neighbour ""Marley"" (Roberts Blossom) and it has that lovely scene on the aircraft when mother Catherine O'Hara realises that it wasn't just the garage doors that they forgot to sort out before they left! A great, and instantly recognisable score from maestro John Williams tops it all off nicely.", + rating = (double?)7.0, + movie_id = 771, + review_id = 771 + }, + new { + author_username = "narrator56", + content = @"Of course we watched this more than 20 years ago, but recently took it out of the library to watch again for a couple of reasons. One, it is ostensibly a holiday movie and we were watching a series of them. Also, a friend had just lost a loved pet and needed a silly movie to take her mind away for a couple of hours. + +This movie fit the bill. It has several laugh out loud scenes, and mildly amusing material surrounding those scenes. The ensemble cast is fine. Catherine O’Hara is a believable mom and I have liked Daniel Stern ever since he couldn’t understand how a VCR works in City Slickers. + +If you are one of those gentle souls like our friend who has difficulty distinguishing between cartoonish fictional violence and reality, you will need to look away a few times. + +It won’t make the regular rotation of our traditional holiday movies, but I am glad we fit it in this year.", + rating = (double?)9.0, + movie_id = 771, + review_id = 771 + } + }; + + foreach (var r in reviewsData) + { + await reviews.Data.Insert( + DataFactory(r), + references: new Dictionary { + { "forMovie", movieIds[(int)r.movie_id] } + }); + } + + // Act + var fun = await reviews.Query.NearText("Fun for the whole family", limit: 2).ToListAsync(TestContext.Current.CancellationToken); + Console.WriteLine(JsonSerializer.Serialize(fun, new JsonSerializerOptions { WriteIndented = true })); + + var disappointed = + await reviews.Query + .NearText( + "Disapointed by this movie", + limit: 2, + references: [new QueryReference("forMovie", ["title"])]) + .ToListAsync(TestContext.Current.CancellationToken); + Console.WriteLine(JsonSerializer.Serialize(disappointed, new JsonSerializerOptions { WriteIndented = true })); + + // Assert + Assert.NotNull(fun); + Assert.Equal(2, fun.Count); + Assert.Equal(0, fun[0]?.References.Count); + Assert.Equal(0, fun[1]?.References.Count); + + + Assert.NotNull(disappointed); + Assert.Equal(2, disappointed.Count); + Assert.Equal(1, disappointed[0]?.References.Count); + Assert.Equal(1, disappointed[1]?.References.Count); + + Assert.True(disappointed[0].References.ContainsKey("forMovie")); + Assert.True(disappointed[1].References.ContainsKey("forMovie")); + Assert.Equal(movieIds[162], disappointed[0].References["forMovie"][0].ID); + Assert.Equal(movieIds[771], disappointed[1].References["forMovie"][0].ID); + } +} diff --git a/src/Weaviate.Client.Tests/Integration/_Integration.cs b/src/Weaviate.Client.Tests/Integration/_Integration.cs new file mode 100644 index 00000000..5551b80b --- /dev/null +++ b/src/Weaviate.Client.Tests/Integration/_Integration.cs @@ -0,0 +1,107 @@ +using Weaviate.Client.Models; + +[assembly: CaptureConsole] +[assembly: CaptureTrace] + +namespace Weaviate.Client.Tests.Integration; + +internal class TestData +{ + public string Name { get; set; } = string.Empty; +} + +internal class TestDataValue +{ + public string Value { get; set; } = string.Empty; +} + +[Collection("BasicTests")] +public partial class BasicTests : IAsyncDisposable +{ + const bool _deleteCollectionsAfterTest = true; + + WeaviateClient _weaviate; + + public BasicTests(ITestOutputHelper output) + { + _weaviate = new WeaviateClient(); + } + + public async ValueTask DisposeAsync() + { + if (_deleteCollectionsAfterTest && TestContext.Current.TestMethod?.MethodName is not null) + { + await _weaviate.Collections.Delete(TestContext.Current.TestMethod!.MethodName); + } + + _weaviate.Dispose(); + } + + async Task> CollectionFactory(string name, + string description, + IList properties, + IList? references = null, + IDictionary? vectorConfig = null) + { + if (string.IsNullOrEmpty(name)) + { + name = TestContext.Current.TestMethod?.MethodName ?? string.Empty; + } + + ArgumentException.ThrowIfNullOrEmpty(name); + + if (vectorConfig is null) + { + vectorConfig = new Dictionary + { + { + "default", new VectorConfig { + Vectorizer = new Dictionary { { "none", new { } } }, + VectorIndexType = "hnsw" + } + } + }; + } + + references = references ?? []; + + var c = new Collection + { + Name = name, + Description = description, + Properties = properties.Concat(references!.Select(p => (Property)p)).ToList(), + VectorConfig = vectorConfig, + }; + + await _weaviate.Collections.Delete(name); + + var collectionClient = await _weaviate.Collections.Create(c); + + return collectionClient; + } + + async Task> CollectionFactory(string name, + string description, + IList properties, + IList? references = null, + IDictionary? vectorConfig = null) + { + return await CollectionFactory(name, description, properties, references, vectorConfig); + } + + WeaviateObject DataFactory(TData value) + { + return new WeaviateObject() + { + Data = value + }; + } + + WeaviateObject DataFactory(TData value, string collectionName) + { + return new WeaviateObject(collectionName) + { + Data = value + }; + } +} diff --git a/src/Weaviate.Client.Tests/Tests.cs b/src/Weaviate.Client.Tests/Tests.cs deleted file mode 100644 index 7b2fcb42..00000000 --- a/src/Weaviate.Client.Tests/Tests.cs +++ /dev/null @@ -1,279 +0,0 @@ -using Weaviate.Client.Models; - -namespace Weaviate.Client.Tests; - -internal class TestData -{ - public string Name { get; set; } = string.Empty; -} - -internal class TestDataValue -{ - public string Value { get; set; } = string.Empty; -} - -[Collection("BasicTests")] -public class WeaviateClientTest : IDisposable -{ - WeaviateClient _weaviate; - - public WeaviateClientTest() - { - _weaviate = new WeaviateClient(); - } - - public void Dispose() - { - _weaviate.Dispose(); - } - - async Task> CollectionFactory(string name, string description, IList properties, IDictionary? vectorConfig = null) - { - if (string.IsNullOrEmpty(name)) - { - name = TestContext.Current.TestMethod?.MethodName ?? string.Empty; - } - - ArgumentException.ThrowIfNullOrEmpty(name); - - if (vectorConfig is null) - { - vectorConfig = new Dictionary - { - { - "default", new VectorConfig { - Vectorizer = new Dictionary { { "none", new { } } }, - VectorIndexType = "hnsw" - } - } - }; - } - - var c = new Collection - { - Name = name, - Description = description, - Properties = properties, - VectorConfig = vectorConfig, - }; - - await _weaviate.Collections.Delete(name); - - var collectionClient = await _weaviate.Collections.Create(c); - - return collectionClient; - } - - async Task> CollectionFactory(string name, string description, IList properties, IDictionary? vectorConfig = null) - { - return await CollectionFactory(name, description, properties, vectorConfig); - } - - WeaviateObject DataFactory(TData value) - { - return new WeaviateObject() - { - Data = value - }; - } - - [Fact] - public async Task TestBasicCollectionCreation() - { - // Arrange - - // Act - var collectionClient = await CollectionFactory("", "Test collection description", [ - Property.Text("Name") - ]); - - // Assert - var collection = await _weaviate.Collections.Use(collectionClient.Name).Get(); - Assert.NotNull(collection); - Assert.Equal("TestBasicCollectionCreation", collection.Name); - Assert.Equal("Test collection description", collection.Description); - } - - [Fact] - public async Task TestBasicObjectCreation() - { - // Arrange - var collectionClient = await CollectionFactory("", "Test collection description", [ - Property.Text("Name") - ]); - - // Act - var id = Guid.NewGuid(); - var obj = await collectionClient.Data.Insert(new WeaviateObject() - { - Data = new TestData { Name = "TestObject" }, - ID = id, - }); - - // Assert - - // Assert object exists - var retrieved = await collectionClient.Query.FetchObjectByID(id); - Assert.NotNull(retrieved); - Assert.Equal(id, retrieved.ID); - Assert.Equal("TestObject", retrieved.Data?.Name); - - // Delete after usage - await collectionClient.Data.Delete(id); - retrieved = await collectionClient.Query.FetchObjectByID(id); - Assert.Null(retrieved); - } - - [Fact] - public async Task TestBasicNearVectorSearch() - { - // Arrange - var collectionClient = await CollectionFactory("", "Test collection description", [ - Property.Text("Name") - ]); - - // Act - await collectionClient.Data.Insert(new WeaviateObject() - { - Data = new TestData { Name = "TestObject1" }, - Vectors = new Dictionary> - { - { "default", new float[] { 0.1f, 0.2f, 0.3f } } - } - }); - - await collectionClient.Data.Insert(new WeaviateObject() - { - Data = new TestData { Name = "TestObject2" }, - Vectors = new Dictionary> - { - { "default", new float[] { 0.3f, 0.4f, 0.5f } } - } - }); - - await collectionClient.Data.Insert(new WeaviateObject() - { - Data = new TestData { Name = "TestObject3" }, - Vectors = new Dictionary> - { - { "default", new float[] { 0.5f, 0.6f, 0.7f } } - } - }); - - // Assert - var retrieved = collectionClient.Query.NearVector(new float[] { 0.1f, 0.2f, 0.3f }); - Assert.NotNull(retrieved); - - await foreach (var obj in retrieved) - { - Assert.Equal("TestObject1", obj.Data!.Name); - break; - } - } - - [Fact] - public async Task TestBasicNearTextSearch() - { - // Arrange - var collectionClient = await CollectionFactory("", "Test collection description", [ - Property.Text("value") - ], new Dictionary - { - { - "default", new VectorConfig - { - Vectorizer = new Dictionary { - { - "text2vec-contextionary", new { - vectorizeClassName = false - } - } - }, - VectorIndexType = "hnsw" - } - } - }); - - string[] values = ["Apple", "Mountain climbing", "apple cake", "cake"]; - var tasks = values.Select(s => new TestDataValue { Value = s }).Select(DataFactory).Select(collectionClient.Data.Insert); - Guid[] guids = await Task.WhenAll(tasks); - var concepts = "hiking"; - - // Act - var retriever = collectionClient.Query.NearText( - "cake", - moveTo: new Move(1.0f, objects: guids[0]), - moveAway: new Move(0.5f, concepts: concepts), - fields: ["value"] - ); - var retrieved = await retriever.ToListAsync(TestContext.Current.CancellationToken); - - // Assert - Assert.NotNull(retrieved); - Assert.Equal(4, retrieved.Count()); - - Assert.Equal(retrieved[0].ID, guids[2]); - Assert.Contains("default", retrieved[0].Vectors.Keys); - Assert.Equal("apple cake", retrieved[0].Data?.Value); - } - - [Fact] - public async Task TestBasicNearTextGroupBySearch() - { - // Arrange - CollectionClient? collectionClient = await CollectionFactory("", "Test collection description", [ - Property.Text("value") - ], new Dictionary - { - { - "default", new VectorConfig - { - Vectorizer = new Dictionary { - { - "text2vec-contextionary", new { - vectorizeClassName = false - } - } - }, - VectorIndexType = "hnsw" - } - } - }); - - string[] values = ["Apple", "Mountain climbing", "apple cake", "cake"]; - var tasks = values.Select(s => new { Value = s }).Select(DataFactory).Select(collectionClient.Data.Insert); - Guid[] guids = await Task.WhenAll(tasks); - - // Act - var retrieved = await collectionClient.Query.NearText( - "cake", - new GroupByConstraint - { - PropertyName = "value", - NumberOfGroups = 2, - ObjectsPerGroup = 100, - } - ); - - // Assert - Assert.NotNull(retrieved.Objects); - Assert.NotNull(retrieved.Groups); - - var retrievedObjects = retrieved.Objects.ToArray(); - - Assert.Equal(2, retrieved.Objects.Count()); - Assert.Equal(2, retrieved.Groups.Count()); - - var obj = await collectionClient.Query.FetchObjectByID(guids[3]); - Assert.NotNull(obj); - Assert.Equal(guids[3], obj.ID); - Assert.Contains("default", obj.Vectors.Keys); - - Assert.Equal(guids[3], retrievedObjects[0].ID); - Assert.Contains("default", retrievedObjects[0].Vectors.Keys); - Assert.Equal("cake", retrievedObjects[0].BelongsToGroup); - Assert.Equal(guids[2], retrievedObjects[1].ID); - Assert.Contains("default", retrievedObjects[1].Vectors.Keys); - Assert.Equal("apple cake", retrievedObjects[1].BelongsToGroup); - } -} diff --git a/src/Weaviate.Client/DataClient.cs b/src/Weaviate.Client/DataClient.cs index 9f5b23b8..ebe83f9a 100644 --- a/src/Weaviate.Client/DataClient.cs +++ b/src/Weaviate.Client/DataClient.cs @@ -1,3 +1,4 @@ +using System.Dynamic; using Weaviate.Client.Models; namespace Weaviate.Client; @@ -13,18 +14,55 @@ public DataClient(CollectionClient collectionClient) _collectionClient = collectionClient; } - public async Task Insert(WeaviateObject data) + public static IDictionary[] MakeBeacons(params Guid[] guids) { + return [ + .. guids.Select(uuid => new Dictionary { { "beacon", $"weaviate://localhost/{uuid}" } }) + ]; + } + + public async Task Insert(WeaviateObject data, Dictionary? references = null) + { + ExpandoObject obj = new ExpandoObject(); + + if (obj is IDictionary propDict) + { + if (references is not null) + { + foreach (var kvp in references) + { + propDict[kvp.Key] = MakeBeacons(kvp.Value); + } + } + + foreach (var property in data.Data?.GetType().GetProperties() ?? []) + { + if (!property.CanRead) + { + continue; + } + + object? value = property.GetValue(data.Data); + + if (value is null) + { + continue; + } + + propDict[property.Name] = value; + } + } + var dto = new Rest.Dto.WeaviateObject() { ID = data.ID ?? Guid.NewGuid(), Class = _collectionName, - Properties = data.Data, + Properties = obj, Vector = data.Vector?.Count == 0 ? null : data.Vector, Vectors = data.Vectors?.Count == 0 ? null : data.Vectors, Additional = data.Additional, - CreationTimeUnix = data.CreationTime.HasValue ? new DateTimeOffset(data.CreationTime.Value).ToUnixTimeMilliseconds() : null, - LastUpdateTimeUnix = data.LastUpdateTime.HasValue ? new DateTimeOffset(data.LastUpdateTime.Value).ToUnixTimeMilliseconds() : null, + CreationTimeUnix = data.Metadata.CreationTime.HasValue ? new DateTimeOffset(data.Metadata.CreationTime.Value).ToUnixTimeMilliseconds() : null, + LastUpdateTimeUnix = data.Metadata.LastUpdateTime.HasValue ? new DateTimeOffset(data.Metadata.LastUpdateTime.Value).ToUnixTimeMilliseconds() : null, Tenant = data.Tenant }; @@ -38,4 +76,18 @@ public async Task Delete(Guid id) await _client.RestClient.DeleteObject(_collectionName, id); } + public async Task ReferenceAdd(Guid from, string fromProperty, Guid to) + { + await _client.RestClient.ReferenceAdd(_collectionName, from, fromProperty, to); + } + + public async Task ReferenceReplace(Guid from, string fromProperty, Guid[] to) + { + await _client.RestClient.ReferenceReplace(_collectionName, from, fromProperty, to); + } + + public async Task ReferenceDelete(Guid from, string fromProperty, Guid to) + { + await _client.RestClient.ReferenceDelete(_collectionName, from, fromProperty, to); + } } \ No newline at end of file diff --git a/src/Weaviate.Client/Extensions.cs b/src/Weaviate.Client/Extensions.cs index 4f656862..aef6e958 100644 --- a/src/Weaviate.Client/Extensions.cs +++ b/src/Weaviate.Client/Extensions.cs @@ -21,8 +21,11 @@ public static WeaviateObject ToWeaviateObject(this WeaviateObject Data = obj, ID = data.ID, Additional = data.Additional, - CreationTime = data.CreationTime, - LastUpdateTime = data.LastUpdateTime, + Metadata = new WeaviateObject.ObjectMetadata + { + CreationTime = data.Metadata.CreationTime, + LastUpdateTime = data.Metadata.LastUpdateTime, + }, Tenant = data.Tenant, Vector = data.Vector, Vectors = data.Vectors, @@ -36,8 +39,11 @@ public static WeaviateObject ToWeaviateObject(this Rest.Dto.WeaviateObject Data = BuildConcreteTypeObjectFromProperties(data.Properties), ID = data.ID, Additional = data.Additional, - CreationTime = data.CreationTimeUnix.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(data.CreationTimeUnix.Value).DateTime : null, - LastUpdateTime = data.LastUpdateTimeUnix.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(data.LastUpdateTimeUnix.Value).DateTime : null, + Metadata = new WeaviateObject.ObjectMetadata + { + CreationTime = data.CreationTimeUnix.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(data.CreationTimeUnix.Value).DateTime : null, + LastUpdateTime = data.LastUpdateTimeUnix.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(data.LastUpdateTimeUnix.Value).DateTime : null, + }, Tenant = data.Tenant, Vector = data.Vector, Vectors = data.Vectors, @@ -53,11 +59,16 @@ public static WeaviateObject ToWeaviateObject(this Models.WeaviateObject d Data = obj, ID = data.ID, Additional = data.Additional, - CreationTime = data.CreationTime, - LastUpdateTime = data.LastUpdateTime, + Metadata = new WeaviateObject.ObjectMetadata + { + CreationTime = data.Metadata.CreationTime, + LastUpdateTime = data.Metadata.LastUpdateTime, + }, + References = data.References, Tenant = data.Tenant, Vector = data.Vector, Vectors = data.Vectors, + Properties = data.Properties, }; } diff --git a/src/Weaviate.Client/Models/MetadataQuery.cs b/src/Weaviate.Client/Models/MetadataQuery.cs new file mode 100644 index 00000000..29ef4661 --- /dev/null +++ b/src/Weaviate.Client/Models/MetadataQuery.cs @@ -0,0 +1,40 @@ +namespace Weaviate.Client.Models; + +[Flags] +public enum MetadataOptions +{ + None = 0, + Vector = 1 << 0, // 2^0 + CreationTime = 1 << 1, // 2^1 + LastUpdateTime = 1 << 2, // 2^2 + Distance = 1 << 3, // 2^3 + Certainty = 1 << 4, // 2^4 + Score = 1 << 5, // 2^5 + ExplainScore = 1 << 6, // 2^6 + IsConsistent = 1 << 7, // 2^7 +} + +public record MetadataQuery(MetadataOptions Options = MetadataOptions.None, HashSet? Vectors = null) +{ + // Implicit conversion from MetadataOptions to MetadataQuery + public static implicit operator MetadataQuery(MetadataOptions options) => new MetadataQuery(options); + + // Implicit conversion from HashSet to MetadataQuery + public static implicit operator MetadataQuery(HashSet vectors) => new MetadataQuery(MetadataOptions.None, vectors); + + // Implicit conversion from (MetadataOptions, HashSet) to MetadataQuery + public static implicit operator MetadataQuery((MetadataOptions options, HashSet vectors) metadata) => new MetadataQuery(metadata.options, metadata.vectors); + + readonly HashSet _vectors = [.. Vectors ?? new HashSet()]; + + public bool Vector => (Options & MetadataOptions.Vector) != 0; + public bool CreationTime => (Options & MetadataOptions.CreationTime) != 0; + public bool LastUpdateTime => (Options & MetadataOptions.LastUpdateTime) != 0; + public bool Distance => (Options & MetadataOptions.Distance) != 0; + public bool Certainty => (Options & MetadataOptions.Certainty) != 0; + public bool Score => (Options & MetadataOptions.Score) != 0; + public bool ExplainScore => (Options & MetadataOptions.ExplainScore) != 0; + public bool IsConsistent => (Options & MetadataOptions.IsConsistent) != 0; + + public HashSet Vectors => _vectors; +} diff --git a/src/Weaviate.Client/Models/Property.cs b/src/Weaviate.Client/Models/Property.cs index 356a453a..7b01d355 100644 --- a/src/Weaviate.Client/Models/Property.cs +++ b/src/Weaviate.Client/Models/Property.cs @@ -1,14 +1,31 @@ + namespace Weaviate.Client.Models; public static class DataType { public static string Text { get; } = "text"; public static string Int { get; } = "int"; + public static string Date { get; } = "date"; public static string Reference(string property) => property.Capitalize(); } +public class ReferenceProperty +{ + public required string Name { get; set; } + public required string TargetCollection { get; set; } + + public static implicit operator Property(ReferenceProperty p) + { + return new Property + { + Name = p.Name, + DataType = { DataType.Reference(p.TargetCollection) } + }; + } +} + public class Property { public required string Name { get; set; } @@ -36,4 +53,23 @@ public static Property Int(string name) DataType = { Models.DataType.Int }, }; } + + + public static Property Date(string name) + { + return new Property + { + Name = name, + DataType = { Models.DataType.Date }, + }; + } + + public static ReferenceProperty Reference(string name, string targetCollection) + { + return new ReferenceProperty + { + Name = name, + TargetCollection = targetCollection + }; + } } diff --git a/src/Weaviate.Client/Models/QueryReference.cs b/src/Weaviate.Client/Models/QueryReference.cs new file mode 100644 index 00000000..6c691c95 --- /dev/null +++ b/src/Weaviate.Client/Models/QueryReference.cs @@ -0,0 +1,17 @@ +namespace Weaviate.Client.Models; + +public record QueryReference +{ + public string LinkOn { get; init; } + public string[] Fields { get; init; } + public MetadataQuery? Metadata { get; init; } + public IList? References { get; init; } + + public QueryReference(string linkOn, string[] fields, MetadataQuery? metadata = null, params QueryReference[]? references) + { + LinkOn = linkOn; + Fields = fields; + Metadata = metadata; + References = references; + } +} \ No newline at end of file diff --git a/src/Weaviate.Client/Models/WeaviateObject.cs b/src/Weaviate.Client/Models/WeaviateObject.cs index 7b239459..d63e1861 100644 --- a/src/Weaviate.Client/Models/WeaviateObject.cs +++ b/src/Weaviate.Client/Models/WeaviateObject.cs @@ -2,28 +2,36 @@ namespace Weaviate.Client.Models; public record WeaviateObject { - public CollectionClient? Collection { get; } + public record ObjectMetadata + { + public DateTime? CreationTime { get; set; } + public DateTime? LastUpdateTime { get; set; } + public double? Distance { get; init; } + public double? Certainty { get; init; } + public double? Score { get; init; } + public string? ExplainScore { get; init; } + public bool? IsConsistent { get; init; } + public double? RerankScore { get; init; } + } public string? CollectionName { get; } public required TData? Data { get; set; } - public Guid? ID { get; set; } + public IDictionary Properties { get; set; } = new Dictionary(); - public IDictionary Additional { get; set; } = new Dictionary(); + public IDictionary> References { get; set; } = new Dictionary>(); - public DateTime? CreationTime { get; set; } + public ObjectMetadata Metadata { get; set; } = new ObjectMetadata(); - public DateTime? LastUpdateTime { get; set; } + public Guid? ID { get; set; } + + public IDictionary Additional { get; set; } = new Dictionary(); public string? Tenant { get; set; } public IDictionary> Vectors { get; set; } = new Dictionary>(); - public WeaviateObject(CollectionClient? collection = null) : this(collection?.Name ?? typeof(TData).Name) - { - Collection = collection; - } public WeaviateObject(string collectionName) { CollectionName = collectionName; @@ -43,10 +51,7 @@ public static IList EmptyVector() public record WeaviateObject : WeaviateObject { - [System.Text.Json.Serialization.JsonConstructor] - public WeaviateObject(CollectionClient? collection = null) : base(collection) { } - - public WeaviateObject(string collectionName) : base(collectionName) { } + public WeaviateObject(string? collectionName = null) : base(collectionName ?? typeof(TData).Name) { } } public record WeaviateObject : WeaviateObject diff --git a/src/Weaviate.Client/QueryClient.cs b/src/Weaviate.Client/QueryClient.cs index 09ffccf8..c2beea79 100644 --- a/src/Weaviate.Client/QueryClient.cs +++ b/src/Weaviate.Client/QueryClient.cs @@ -18,9 +18,9 @@ public QueryClient(CollectionClient collectionClient) #region Objects - public async IAsyncEnumerable> List(uint? limit = null) + public async IAsyncEnumerable> List(uint? limit = null, IList? references = null, MetadataQuery? metadata = null) { - var list = await _client.GrpcClient.FetchObjects(_collectionName, limit: limit); + var list = await _client.GrpcClient.FetchObjects(_collectionName, limit: limit, reference: references, metadata: metadata); foreach (var data in list.ToObjects()) { @@ -28,9 +28,9 @@ public async IAsyncEnumerable> List(uint? limit = null) } } - public async Task?> FetchObjectByID(Guid id) + public async Task?> FetchObjectByID(Guid id, IList? references = null, MetadataQuery? metadata = null) { - var reply = await _client.GrpcClient.FetchObjects(_collectionName, Filter.WithID(id)); + var reply = await _client.GrpcClient.FetchObjects(_collectionName, filter: Filter.WithID(id), reference: references, metadata: metadata); var data = reply.FirstOrDefault(); @@ -42,9 +42,9 @@ public async IAsyncEnumerable> List(uint? limit = null) return data.ToWeaviateObject(); } - public async IAsyncEnumerable> FetchObjectsByIDs(ISet ids, uint? limit = null) + public async IAsyncEnumerable> FetchObjectsByIDs(ISet ids, uint? limit = null, IList? references = null, MetadataQuery? metadata = null) { - var list = await _client.GrpcClient.FetchObjects(_collectionName, limit: limit, filter: Filter.WithIDs(ids)); + var list = await _client.GrpcClient.FetchObjects(_collectionName, limit: limit, filter: Filter.WithIDs(ids), reference: references, metadata: metadata); foreach (var r in list.Select(x => x.ToWeaviateObject())) { @@ -55,8 +55,15 @@ public async IAsyncEnumerable> FetchObjectsByIDs(ISet> NearText(string text, float? distance = null, float? certainty = null, uint? limit = null, string[]? fields = null, - string[]? metadata = null, Move? moveTo = null, Move? moveAway = null) + public async IAsyncEnumerable> NearText(string text, + float? distance = null, + float? certainty = null, + uint? limit = null, + string[]? fields = null, + IList? references = null, + MetadataQuery? metadata = null, + Move? moveTo = null, + Move? moveAway = null) { var results = await _client.GrpcClient.SearchNearText( @@ -65,6 +72,8 @@ await _client.GrpcClient.SearchNearText( distance: distance, certainty: certainty, limit: limit, + reference: references, + metadata: metadata, moveTo: moveTo, moveAway: moveAway ); @@ -75,9 +84,14 @@ await _client.GrpcClient.SearchNearText( } } - public async Task NearText(string text, Models.GroupByConstraint groupBy, float? distance = null, - float? certainty = null, uint? limit = null, string[]? fields = null, - string[]? metadata = null) + public async Task NearText(string text, + Models.GroupByConstraint groupBy, + float? distance = null, + float? certainty = null, + uint? limit = null, + string[]? fields = null, + IList? references = null, + MetadataQuery? metadata = null) { var results = await _client.GrpcClient.SearchNearText( @@ -86,13 +100,21 @@ await _client.GrpcClient.SearchNearText( groupBy, distance: distance, certainty: certainty, - limit: limit + limit: limit, + reference: references, + metadata: metadata ); return results; } - public async IAsyncEnumerable> NearVector(float[] vector, float? distance = null, float? certainty = null, uint? limit = null, string[]? fields = null, string[]? metadata = null) + public async IAsyncEnumerable> NearVector(float[] vector, + float? distance = null, + float? certainty = null, + uint? limit = null, + string[]? fields = null, + IList? references = null, + MetadataQuery? metadata = null) { var results = await _client.GrpcClient.SearchNearVector( @@ -100,7 +122,9 @@ await _client.GrpcClient.SearchNearVector( vector, distance: distance, certainty: certainty, - limit: limit + limit: limit, + reference: references, + metadata: metadata ); foreach (var r in results) @@ -109,9 +133,14 @@ await _client.GrpcClient.SearchNearVector( } } - public async Task NearVector(float[] vector, GroupByConstraint groupBy, float? distance = null, - float? certainty = null, uint? limit = null, string[]? fields = null, - string[]? metadata = null) + public async Task NearVector(float[] vector, + GroupByConstraint groupBy, + float? distance = null, + float? certainty = null, + uint? limit = null, + string[]? fields = null, + IList? references = null, + MetadataQuery? metadata = null) { var results = await _client.GrpcClient.SearchNearVector( @@ -120,11 +149,32 @@ await _client.GrpcClient.SearchNearVector( groupBy, distance: distance, certainty: certainty, - limit: limit + limit: limit, + reference: references, + metadata: metadata ); return results; } + public async Task>> BM25(string query, + string[]? searchFields = null, + string[]? fields = null, + IList? references = null, + MetadataQuery? metadata = null) + { + var results = + await _client.GrpcClient.SearchBM25( + _collectionClient.Name, + query: query, + searchFields: searchFields, + fields: fields, + reference: references, + metadata: metadata + ); + + return results.Select(r => r.ToWeaviateObject()); + } + #endregion } \ No newline at end of file diff --git a/src/Weaviate.Client/Rest/Client.cs b/src/Weaviate.Client/Rest/Client.cs index 70523391..08387fa7 100644 --- a/src/Weaviate.Client/Rest/Client.cs +++ b/src/Weaviate.Client/Rest/Client.cs @@ -36,15 +36,15 @@ protected override async Task SendAsync(HttpRequestMessage _log($"Response: {response.StatusCode}"); - if (response.Content != null) + foreach (var header in response.Headers) { - var responseContent = await response.Content.ReadAsStringAsync(); - _log($"Response Content: {responseContent}"); + _log($"Response Header: {header.Key}: {string.Join(", ", header.Value)}"); } - foreach (var header in response.Headers) + if (response.Content != null) { - _log($"Response Header: {header.Key}: {string.Join(", ", header.Value)}"); + var responseContent = await response.Content.ReadAsStringAsync(); + _log($"Response Content: {responseContent}"); } return response; @@ -116,39 +116,53 @@ public void Dispose() } } - private void ValidateResponseStatusCode(HttpResponseMessage response, ExpectedStatusCodes expectedStatusCodes) + private HttpResponseMessage ValidateResponseStatusCode(HttpResponseMessage response, ExpectedStatusCodes expectedStatusCodes) { if (!expectedStatusCodes.Ok.Contains((int)response.StatusCode)) { throw new HttpRequestException($"Unexpected status code: {response.StatusCode}. Expected one of: {string.Join(", ", expectedStatusCodes.Ok)}"); } + + return response; } internal async Task GetAsync(string requestUri, ExpectedStatusCodes expectedStatusCodes) { var response = await _httpClient.GetAsync(requestUri); - ValidateResponseStatusCode(response, expectedStatusCodes); - - return response; + return ValidateResponseStatusCode(response, expectedStatusCodes); } internal async Task DeleteAsync(string requestUri, ExpectedStatusCodes expectedStatusCodes) { var response = await _httpClient.DeleteAsync(requestUri); - ValidateResponseStatusCode(response, expectedStatusCodes); - - return response; + return ValidateResponseStatusCode(response, expectedStatusCodes); } internal async Task PostAsJsonAsync(string? requestUri, TValue value, ExpectedStatusCodes expectedStatusCodes) { var response = await _httpClient.PostAsJsonAsync(requestUri, value); - ValidateResponseStatusCode(response, expectedStatusCodes); + return ValidateResponseStatusCode(response, expectedStatusCodes); + } - return response; + internal async Task PutAsJsonAsync(string? requestUri, TValue value, ExpectedStatusCodes expectedStatusCodes) + { + var response = await _httpClient.PutAsJsonAsync(requestUri, value); + + return ValidateResponseStatusCode(response, expectedStatusCodes); + } + + internal async Task DeleteAsJsonAsync(string? requestUri, TValue value, ExpectedStatusCodes expectedStatusCodes) + { + + var request = new HttpRequestMessage(HttpMethod.Delete, requestUri); + request.Content = JsonContent.Create(value, mediaType: null, null); + + var response = await _httpClient.SendAsync(request); + + return ValidateResponseStatusCode(response, expectedStatusCodes); } } @@ -246,4 +260,35 @@ internal async Task DeleteObject(string collectionName, Guid id) { await _httpClient.DeleteAsync($"objects/{collectionName}/{id}", new ExpectedStatusCodes(new List { 204, 404 }, "delete object")); } + + internal async Task ReferenceAdd(string collectionName, Guid from, string fromProperty, Guid to) + { + var path = $"objects/{collectionName}/{from}/references/{fromProperty}"; + + var beacons = DataClient.MakeBeacons(to); + var reference = beacons.First(); + + var response = await _httpClient.PostAsJsonAsync(path, reference, new ExpectedStatusCodes(new List { 200 }, "reference add")); + } + + internal async Task ReferenceReplace(string collectionName, Guid from, string fromProperty, Guid[] to) + { + var path = $"objects/{collectionName}/{from}/references/{fromProperty}"; + + var beacons = DataClient.MakeBeacons(to); + var reference = beacons; + + var response = await _httpClient.PutAsJsonAsync(path, reference, new ExpectedStatusCodes(new List { 200 }, "reference replace")); + } + + + internal async Task ReferenceDelete(string collectionName, Guid from, string fromProperty, Guid to) + { + var path = $"objects/{collectionName}/{from}/references/{fromProperty}"; + + var beacons = DataClient.MakeBeacons(to); + var reference = beacons.First(); + + var response = await _httpClient.DeleteAsJsonAsync(path, reference, new ExpectedStatusCodes(new List { 200 }, "reference delete")); + } } \ No newline at end of file diff --git a/src/Weaviate.Client/gRPC/Client.cs b/src/Weaviate.Client/gRPC/Client.cs index d85aa480..5be6c70e 100644 --- a/src/Weaviate.Client/gRPC/Client.cs +++ b/src/Weaviate.Client/gRPC/Client.cs @@ -30,7 +30,7 @@ private static IList buildListFromListValue(ListValue list) case ListValue.KindOneofCase.BoolValues: return list.BoolValues.Values; case ListValue.KindOneofCase.ObjectValues: - return list.ObjectValues.Values.Select(v => buildObjectFromProperties(v)).ToList(); + return list.ObjectValues.Values.Select(v => MakeNonRefs(v)).ToList(); case ListValue.KindOneofCase.DateValues: return list.DateValues.Values; // TODO Parse dates here? case ListValue.KindOneofCase.UuidValues: @@ -45,9 +45,15 @@ private static IList buildListFromListValue(ListValue list) } } - private static ExpandoObject buildObjectFromProperties(Properties result) + private static ExpandoObject MakeNonRefs(Properties result) { var eoBase = new ExpandoObject(); + + if (result is null) + { + return eoBase; + } + var eo = eoBase as IDictionary; foreach (var r in result.Fields) @@ -68,7 +74,7 @@ private static ExpandoObject buildObjectFromProperties(Properties result) eo[r.Key] = r.Value.BoolValue; break; case Value.KindOneofCase.ObjectValue: - eo[r.Key] = buildObjectFromProperties(r.Value.ObjectValue) ?? new object { }; + eo[r.Key] = MakeNonRefs(r.Value.ObjectValue) ?? new object { }; break; case Value.KindOneofCase.ListValue: eo[r.Key] = buildListFromListValue(r.Value.ListValue); diff --git a/src/Weaviate.Client/gRPC/Search.cs b/src/Weaviate.Client/gRPC/Search.cs index 266e8886..f3f5d595 100644 --- a/src/Weaviate.Client/gRPC/Search.cs +++ b/src/Weaviate.Client/gRPC/Search.cs @@ -1,3 +1,4 @@ +using Google.Protobuf.Collections; using Weaviate.Client.Models; using Weaviate.V1; @@ -7,8 +8,23 @@ namespace Weaviate.Client.Grpc; public partial class WeaviateGrpcClient { - internal SearchRequest BaseSearchRequest(string collection, Filters? filter = null, uint? limit = null, GroupByConstraint? groupBy = null) + internal SearchRequest BaseSearchRequest(string collection, Filters? filter = null, uint? limit = null, GroupByConstraint? groupBy = null, MetadataQuery? metadata = null, IList? reference = null, string[]? fields = null) { + var metadataRequest = new MetadataRequest() + { + Uuid = true, + Vector = metadata?.Vector ?? false, + LastUpdateTimeUnix = metadata?.LastUpdateTime ?? false, + CreationTimeUnix = metadata?.CreationTime ?? false, + Certainty = metadata?.Certainty ?? false, + Distance = metadata?.Distance ?? false, + Score = metadata?.Score ?? false, + ExplainScore = metadata?.ExplainScore ?? false, + IsConsistent = metadata?.IsConsistent ?? false + }; + + metadataRequest.Vectors.AddRange(metadata?.Vectors.ToArray() ?? []); + return new SearchRequest() { Collection = collection, @@ -23,11 +39,80 @@ internal SearchRequest BaseSearchRequest(string collection, Filters? filter = nu NumberOfGroups = Convert.ToInt32(groupBy.NumberOfGroups), ObjectsPerGroup = Convert.ToInt32(groupBy.ObjectsPerGroup), } : null, + Metadata = metadataRequest, + Properties = MakePropsRequest(fields, reference) + }; + } + + private PropertiesRequest? MakePropsRequest(string[]? fields, IList? reference) + { + if (fields is null && reference is null) return null; + + var req = new PropertiesRequest(); + + if (fields is not null) + { + req.NonRefProperties.AddRange(fields); + } + else + { + req.ReturnAllNonrefProperties = true; + } + + foreach (var r in reference ?? []) + { + if (reference is not null) + { + req.RefProperties.Add(MakeRefPropsRequest(r)); + } + } + + return req; + } + + private RefPropertiesRequest? MakeRefPropsRequest(QueryReference? reference) + { + if (reference is null) return null; + + return new RefPropertiesRequest() + { Metadata = new MetadataRequest() { Uuid = true, - Vector = true, - Vectors = { "default" } + LastUpdateTimeUnix = reference.Metadata?.LastUpdateTime ?? false, + CreationTimeUnix = reference.Metadata?.CreationTime ?? false, + Certainty = reference.Metadata?.Certainty ?? false, + Distance = reference.Metadata?.Distance ?? false, + Score = reference.Metadata?.Score ?? false, + ExplainScore = reference.Metadata?.ExplainScore ?? false, + IsConsistent = reference.Metadata?.IsConsistent ?? false + }, + Properties = MakePropsRequest(reference.Fields, reference.References), + ReferenceProperty = reference.LinkOn, + }; + } + + private static WeaviateObject BuildObjectFromResult(MetadataResult metadata, Properties properties, RepeatedField references, string collection) + { + var data = MakeNonRefs(properties); + + return new Models.WeaviateObject(collection) + { + ID = !string.IsNullOrEmpty(metadata.Id) ? Guid.Parse(metadata.Id) : Guid.Empty, + Vector = metadata.Vector, + Vectors = metadata.Vectors.ToDictionary(v => v.Name, v => (IList)v.VectorBytes.FromByteString().ToList()), + Data = data, + Properties = data as IDictionary, + References = MakeRefs(references), + Metadata = new WeaviateObject.ObjectMetadata() + { + LastUpdateTime = metadata.LastUpdateTimeUnixPresent ? DateTimeOffset.FromUnixTimeMilliseconds(metadata.LastUpdateTimeUnix).DateTime : null, + CreationTime = metadata.CreationTimeUnixPresent ? DateTimeOffset.FromUnixTimeMilliseconds(metadata.CreationTimeUnix).DateTime : null, + Certainty = metadata.CertaintyPresent ? metadata.Certainty : null, + Distance = metadata.DistancePresent ? metadata.Distance : null, + Score = metadata.ScorePresent ? metadata.Score : null, + ExplainScore = metadata.ExplainScorePresent ? metadata.ExplainScore : null, + IsConsistent = metadata.IsConsistentPresent ? metadata.IsConsistent : null } }; } @@ -39,13 +124,19 @@ private static IEnumerable BuildResult(string collection, Search return []; } - return reply.Results.Select(result => new Models.WeaviateObject(collection) + return reply.Results.Select(r => BuildObjectFromResult(r.Metadata, r.Properties.NonRefProps, r.Properties.RefProps, collection)); + } + + private static IDictionary> MakeRefs(RepeatedField refProps) + { + var result = new Dictionary>(); + + foreach (var refProp in refProps) { - ID = Guid.Parse(result.Metadata.Id), - Vector = result.Metadata.Vector, - Vectors = result.Metadata.Vectors.ToDictionary(v => v.Name, v => (IList)v.VectorBytes.FromByteString().ToList()), - Data = buildObjectFromProperties(result.Properties.NonRefProps), - }); + result[refProp.PropName] = refProp.Properties.Select(p => BuildObjectFromResult(p.Metadata, p.NonRefProps, p.RefProps, p.TargetCollection)).ToList(); + } + + return result; } private static Models.GroupByResult BuildGroupByResult(string collection, SearchReply reply) @@ -63,7 +154,7 @@ private static Models.GroupByResult BuildGroupByResult(string collection, Search ID = Guid.Parse(obj.Metadata.Id), Vector = obj.Metadata.Vector, Vectors = obj.Metadata.Vectors.ToDictionary(v => v.Name, v => (IList)v.VectorBytes.FromByteString().ToList()), - Data = buildObjectFromProperties(obj.Properties.NonRefProps), + Data = MakeNonRefs(obj.Properties.NonRefProps), BelongsToGroup = v.Name, }).ToArray() }); @@ -144,18 +235,31 @@ private static void BuildNearVector(float[] vector, float? distance, float? cert } } - internal async Task> FetchObjects(string collection, Filters? filter = null, uint? limit = null) + private void BuildBM25(SearchRequest request, string query, string[]? properties = null) { - var req = BaseSearchRequest(collection, filter, limit); + request.Bm25Search = new BM25() + { + Query = query, + }; + + if (properties is not null) + { + request.Bm25Search.Properties.AddRange(properties); + } + } + + internal async Task> FetchObjects(string collection, Filters? filter = null, uint? limit = null, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) + { + var req = BaseSearchRequest(collection, filter, limit, fields: fields, metadata: metadata, reference: reference); SearchReply? reply = await _grpcClient.SearchAsync(req); return BuildResult(collection, reply); } - public async Task> SearchNearVector(string collection, float[] vector, float? distance = null, float? certainty = null, uint? limit = null) + public async Task> SearchNearVector(string collection, float[] vector, float? distance = null, float? certainty = null, uint? limit = null, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) { - var request = BaseSearchRequest(collection, filter: null, limit: limit); + var request = BaseSearchRequest(collection, filter: null, limit: limit, fields: fields, metadata: metadata, reference: reference); BuildNearVector(vector, distance, certainty, request); @@ -164,9 +268,9 @@ public async Task> SearchNearVector(string collectio return BuildResult(collection, reply); } - internal async Task> SearchNearText(string collection, string query, float? distance, float? certainty, uint? limit, Move? moveTo, Move? moveAway) + internal async Task> SearchNearText(string collection, string query, float? distance, float? certainty, uint? limit, Move? moveTo, Move? moveAway, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) { - var request = BaseSearchRequest(collection, filter: null, limit: limit); + var request = BaseSearchRequest(collection, filter: null, limit: limit, fields: fields, metadata: metadata, reference: reference); BuildNearText(query, distance, certainty, request, moveTo, moveAway); @@ -175,9 +279,9 @@ internal async Task> SearchNearText(string collectio return BuildResult(collection, reply); } - public async Task SearchNearVector(string collection, float[] vector, GroupByConstraint groupBy, float? distance = null, float? certainty = null, uint? limit = null) + public async Task SearchNearVector(string collection, float[] vector, GroupByConstraint groupBy, float? distance = null, float? certainty = null, uint? limit = null, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) { - var request = BaseSearchRequest(collection, filter: null, limit: limit, groupBy: groupBy); + var request = BaseSearchRequest(collection, filter: null, limit: limit, groupBy: groupBy, fields: fields, metadata: metadata, reference: reference); BuildNearVector(vector, distance, certainty, request); @@ -186,9 +290,9 @@ internal async Task> SearchNearText(string collectio return BuildGroupByResult(collection, reply); } - internal async Task SearchNearText(string collection, string query, GroupByConstraint groupBy, float? distance, float? certainty, uint? limit) + internal async Task SearchNearText(string collection, string query, GroupByConstraint groupBy, float? distance, float? certainty, uint? limit, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) { - var request = BaseSearchRequest(collection, filter: null, limit: limit, groupBy: groupBy); + var request = BaseSearchRequest(collection, filter: null, limit: limit, groupBy: groupBy, fields: fields, metadata: metadata, reference: reference); BuildNearText(query, distance, certainty, request, moveTo: null, moveAway: null); @@ -196,4 +300,15 @@ internal async Task> SearchNearText(string collectio return BuildGroupByResult(collection, reply); } + + internal async Task> SearchBM25(string collection, string query, string[]? searchFields, string[]? fields = null, IList? reference = null, MetadataQuery? metadata = null) + { + var request = BaseSearchRequest(collection, filter: null, limit: null, groupBy: null, fields: fields, metadata: metadata, reference: reference); + + BuildBM25(request, query, properties: searchFields); + + SearchReply? reply = await _grpcClient.SearchAsync(request); + + return BuildResult(collection, reply); + } }