From d471cd3578a7f08450e5331669e9e08c4817266c Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Mon, 15 Sep 2025 15:26:31 +0200 Subject: [PATCH 1/2] Fix search + missing senses --- .../FwLite/LcmCrdt/Data/MiniLcmRepository.cs | 26 ++++--- .../MiniLcm.Tests/QueryEntryTestsBase.cs | 75 +++++++++++++++++++ 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs b/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs index 712e7ea685..3da29ef833 100644 --- a/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs +++ b/backend/FwLite/LcmCrdt/Data/MiniLcmRepository.cs @@ -157,6 +157,20 @@ public async IAsyncEnumerable GetEntries( string? query, FilterQueryOptions options) { + if (options.Exemplar is not null) + { + var ws = (await GetWritingSystem(options.Exemplar.WritingSystem, WritingSystemType.Vernacular))?.WsId; + if (ws is null) + throw new NullReferenceException($"writing system {options.Exemplar.WritingSystem} not found"); + queryable = queryable.WhereExemplar(ws.Value, options.Exemplar.Value); + } + + if (options.Filter?.GridifyFilter != null) + { + // Do this BEFORE doing the FTS, which returns an expression that confuses the gridify query + queryable = queryable.ApplyFiltering(options.Filter.GridifyFilter, config.Value.Mapper); + } + bool sortingHandled = false; if (!string.IsNullOrEmpty(query)) { @@ -176,18 +190,6 @@ public async IAsyncEnumerable GetEntries( } } - if (options.Exemplar is not null) - { - var ws = (await GetWritingSystem(options.Exemplar.WritingSystem, WritingSystemType.Vernacular))?.WsId; - if (ws is null) - throw new NullReferenceException($"writing system {options.Exemplar.WritingSystem} not found"); - queryable = queryable.WhereExemplar(ws.Value, options.Exemplar.Value); - } - - if (options.Filter?.GridifyFilter != null) - { - queryable = queryable.ApplyFiltering(options.Filter.GridifyFilter, config.Value.Mapper); - } return (queryable, sortingHandled); } diff --git a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs index c71a70f9b9..c9943f1bb1 100644 --- a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs @@ -100,6 +100,14 @@ public async Task CanFilterToMissingSenses() results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Apple, Null_LexemeForm); } + [Fact] + public async Task CanFilterToMissingSenses_AndSearch() + { + var results = await Api.SearchEntries(Apple, new(Filter: new() { GridifyFilter = "Senses=null" })).ToArrayAsync(); + //using distinct since there may be 2 null lexeme forms but only on FLEx due to the null lexeme form + results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Apple); + } + [Fact] public async Task CanFilterToNotMissingSenses() { @@ -107,6 +115,13 @@ public async Task CanFilterToNotMissingSenses() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Kiwi, Peach, Banana); } + [Fact] + public async Task CanFilterToNotMissingSenses_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "Senses!=null" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterToMissingPartOfSpeech() { @@ -115,6 +130,14 @@ public async Task CanFilterToMissingPartOfSpeech() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); } + [Fact] + public async Task CanFilterToMissingPartOfSpeech_AndSearch() + { + var results = await Api.SearchEntries(Peach, new(Filter: new() { GridifyFilter = "Senses.PartOfSpeechId=" })).ToArrayAsync(); + //does not include entries with no senses + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); + } + [Fact] public async Task CanFilterToMissingExamples() { @@ -124,6 +147,15 @@ public async Task CanFilterToMissingExamples() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach, Banana); } + [Fact] + public async Task CanFilterToMissingExamples_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "Senses.ExampleSentences=null" })).ToArrayAsync(); + //Senses.ExampleSentences=null matches entries which have senses but no examples + //it does not include Apple because it has no senses, to include it a filter Senses=null is needed + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterToMissingSemanticDomains() { @@ -131,6 +163,13 @@ public async Task CanFilterToMissingSemanticDomains() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); } + [Fact] + public async Task CanFilterToMissingSemanticDomains_AndSearch() + { + var results = await Api.SearchEntries(Peach, new(Filter: new() { GridifyFilter = "Senses.SemanticDomains=null" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); + } + [Fact] public async Task CanFilterToMissingSemanticDomainsWithEmptyArray() { @@ -153,6 +192,14 @@ public async Task CanFilterToMissingComplexFormTypes() results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Apple, Banana, Kiwi, Null_LexemeForm); } + [Fact] + public async Task CanFilterToMissingComplexFormTypes_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "ComplexFormTypes=null" })).ToArrayAsync(); + //using distinct since there may be 2 null lexeme forms but only on FLEx due to the null lexeme form + results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterToMissingComplexFormTypesWithEmptyArray() { @@ -189,6 +236,13 @@ public async Task CanFilterLexemeFormContains() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); } + [Fact] + public async Task CanFilterLexemeFormContains_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "LexemeForm[en]=*nan" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterGlossNull() { @@ -205,6 +259,13 @@ public async Task CanFilterGlossEmptyOrNull() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); } + [Fact] + public async Task CanFilterGlossEmptyOrNull_AndSearch() + { + var results = await Api.SearchEntries(Peach, new(Filter: new() { GridifyFilter = "Senses.Gloss[en]=" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Peach); + } + [Fact] public async Task CanFilterGlossEqualsFruit() { @@ -212,6 +273,13 @@ public async Task CanFilterGlossEqualsFruit() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana, Kiwi); } + [Fact] + public async Task CanFilterGlossEqualsFruit_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "Senses.Gloss[en]=Fruit" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterLexemeContainsAAndNoComplexFormTypes() { @@ -226,6 +294,13 @@ public async Task CanFilterExampleSentenceText() results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); } + [Fact] + public async Task CanFilterExampleSentenceText_AndSearch() + { + var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "Senses.ExampleSentences.Sentence[en]=*phone" })).ToArrayAsync(); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); + } + [Fact] public async Task CanFilterToExampleSentenceWithMissingSentence() { From 6ae712d8ff9821d0c6fbd4e2d1832b1e71868d93 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Mon, 15 Sep 2025 15:46:26 +0200 Subject: [PATCH 2/2] Remove irrelevant distinct checks --- backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs index c9943f1bb1..28078d9ebc 100644 --- a/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs +++ b/backend/FwLite/MiniLcm.Tests/QueryEntryTestsBase.cs @@ -104,8 +104,7 @@ public async Task CanFilterToMissingSenses() public async Task CanFilterToMissingSenses_AndSearch() { var results = await Api.SearchEntries(Apple, new(Filter: new() { GridifyFilter = "Senses=null" })).ToArrayAsync(); - //using distinct since there may be 2 null lexeme forms but only on FLEx due to the null lexeme form - results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Apple); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Apple); } [Fact] @@ -196,8 +195,7 @@ public async Task CanFilterToMissingComplexFormTypes() public async Task CanFilterToMissingComplexFormTypes_AndSearch() { var results = await Api.SearchEntries(Banana, new(Filter: new() { GridifyFilter = "ComplexFormTypes=null" })).ToArrayAsync(); - //using distinct since there may be 2 null lexeme forms but only on FLEx due to the null lexeme form - results.Select(e => e.LexemeForm["en"]).Distinct().Should().BeEquivalentTo(Banana); + results.Select(e => e.LexemeForm["en"]).Should().BeEquivalentTo(Banana); } [Fact]