native-repo fuzzy search: fixed NOT treatment + more tests

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/filtering/ExtensionItemFilterProcessor.java

@@ -56,6 +56,14 @@
 /**
  * Filter processor for extension items stored in JSONB.
  * This takes care of any supported type, scalar or array, and handles any operation.
+ *
+ * NOTE about NOT treatment:
+ * We use the same not treatment for extensions like for other columns resulting in conditions like:
+ * `not (u.ext->>'1510' < ? and u.ext->>'1510' is not null)`
+ * One might think that the part after AND can be replaced with u.ext ? '1510' to benefit from the GIN index.
+ * But `NOT (u.ext ? '...')` is *not* fully complement to the `u.ext ? '...'` (without NOT).
+ * It is only fully complement if additional `AND u.ext is not null` is added in which case the index will not be used either.
+ * So instead of adding special treatment code for extensions, we just reuse existing predicateWithNotTreated methods.
  */
 public class ExtensionItemFilterProcessor extends ItemValueFilterProcessor<ValueFilter<?, ?>> {
 

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoIdentityDataTest.java

@@ -120,7 +120,7 @@ public void test115GetUserWithRetrieveOptions() throws CommonException {
     }
 
     @Test
-    public void test200SearchUsingFuzzyMatching() throws CommonException {
+    public void test200SearchUsingNonFuzzyMatching() throws CommonException {
         given("query by focus identity normalized item");
         ItemName familyNameQName = new ItemName(SchemaConstants.NS_C, "familyName");
         var def = PrismContext.get().definitionFactory()
@@ -138,27 +138,53 @@ public void test200SearchUsingFuzzyMatching() throws CommonException {
 
         when("search is executed without any get options");
         OperationResult result = createOperationResult();
-        SearchResultList<PrismObject<UserType>> users = repositoryService.searchObjects(UserType.class, query, null, result);
+        SearchResultList<UserType> users = searchObjects(UserType.class, query, result);
         assertThatOperationResult(result).isSuccess();
 
         then("result contains found user but identities are incomplete");
         assertThat(users).singleElement()
-                .matches(u -> u.findContainer(FocusType.F_IDENTITIES).isIncomplete());
+                .matches(u -> u.asPrismObject().findContainer(FocusType.F_IDENTITIES).isIncomplete());
 
         when("search is executed with retrieve items options");
         result = createOperationResult();
-        users = repositoryService.searchObjects(UserType.class, query, getWithIdentitiesOptions, result);
+        users = searchObjects(UserType.class, query, result, getWithIdentitiesOptions);
         assertThatOperationResult(result).isSuccess();
 
         then("result contains round user with complete identities container");
         assertThat(users).singleElement()
-                .matches(u -> !u.findContainer(FocusType.F_IDENTITIES).isIncomplete()); // is complete this time
-        UserType user = users.get(0).asObjectable();
+                .matches(u -> !u.asPrismObject().findContainer(FocusType.F_IDENTITIES).isIncomplete()); // is complete this time
+        UserType user = users.get(0);
         List<FocusIdentityType> identities = user.getIdentities().getIdentity();
         assertThat(identities).hasSize(2);
     }
 
-    // TODO search fuzzy
+    @Test
+    public void test210SearchUsingFuzzyMatching() throws CommonException {
+        given("query by focus identity normalized item using levenshtein");
+        ItemName familyNameQName = new ItemName(SchemaConstants.NS_C, "familyName");
+        var def = PrismContext.get().definitionFactory()
+                .createPropertyDefinition(familyNameQName, DOMUtil.XSD_STRING, null, null);
+
+        ObjectQuery query = PrismContext.get().queryFor(UserType.class)
+                .itemWithDef(def,
+                        UserType.F_IDENTITIES,
+                        FocusIdentitiesType.F_IDENTITY,
+                        FocusIdentityType.F_ITEMS,
+                        FocusIdentityItemsType.F_NORMALIZED,
+                        familyNameQName)
+                .fuzzyString("gren").levenshteinInclusive(3)
+                .build();
+
+        when("search is executed");
+        OperationResult result = createOperationResult();
+        SearchResultList<UserType> users = searchObjects(UserType.class, query, result);
+        assertThatOperationResult(result).isSuccess();
+
+        then("result contains found user");
+        assertThat(users).hasSize(1);
+        UserType user = users.get(0);
+        assertThat(user.getOid()).isEqualTo(userOid);
+    }
 
     @Test
     public void test300ModifyAddIdentityContainer() throws CommonException {

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSearchTest.java

@@ -2724,8 +2724,31 @@ public void test982SearchRoleReferencedByUserAssignmentWithComplexFilter() throw
     @Test
     public void test982SearchRoleReferencedByUserAssignmentWithComplexFilterNoMatch() throws SchemaException {
         searchObjectTest("referenced by an assignment of the user specified by complex filter (no match)", RoleType.class,
-//                f -> f.referencedBy(AssignmentType.class, AssignmentType.F_TARGET_REF)
-//                        .ownedBy(UserType.class, F_ASSIGNMENT)
+                /*
+                This filter followed byt the .block() part is logically equivalent to the one used in code:
+                f -> f.referencedBy(AssignmentType.class, AssignmentType.F_TARGET_REF)
+                        .ownedBy(UserType.class, F_ASSIGNMENT)
+
+                Both produce similar select, just with different nesting of EXISTS and WHERE.
+                For the commented code we're searching for role referenced by assignment owned by user
+                (exactly what the fluent API says):
+                select r.oid, r.fullObject from m_role r
+                where exists (select 1 from m_assignment a
+                    where a.targetRefTargetOid = r.oid
+                        and exists (select 1 from m_user u
+                            where u.oid = a.ownerOid and a.containerType = ?
+                                and (not u.costCenter is null
+                                    and (u.policySituations = '{}' OR u.policySituations is null))))
+
+                And the select for the code from test, where we search role referenced from user's assignment:
+
+                select r.oid, r.fullObject from m_role r
+                where exists (select 1 from m_user u
+                    where exists (select 1 from m_assignment a
+                        where u.oid = a.ownerOid and a.containerType = ?
+                            and a.targetRefTargetOid = r.oid)
+                        and (not u.costCenter is null and (u.policySituations = '{}' OR u.policySituations is null)))
+                */
                 f -> f.referencedBy(UserType.class,
                                 ItemPath.create(UserType.F_ASSIGNMENT, AssignmentType.F_TARGET_REF))
                         .block()
@@ -2764,20 +2787,42 @@ public void test991SearchObjectWithStringIgnoreCaseWithoutNamespace()
     }
 
     @Test
-    public void fuzzyStringSearchTest() throws SchemaException {
-        searchUsersTest("With levenshtein",
-                f -> f.item(UserType.F_EMPLOYEE_NUMBER).fuzzyString("User1").levenshtein(2, true),
+    public void test992FuzzyStringSearch() throws SchemaException {
+        searchUsersTest("with levenshtein filter against string item",
+                f -> f.item(UserType.F_EMPLOYEE_NUMBER)
+                        .fuzzyString("User1").levenshteinInclusive(2),
                 user1Oid);
 
-        searchUsersTest("With levenshtein in extension",
-                f -> f.item(UserType.F_EXTENSION, new ItemName("string")).fuzzyString("string_value").levenshtein(2, true),
+        searchUsersTest("with levenshtein filter against extension item",
+                f -> f.item(UserType.F_EXTENSION, new ItemName("string"))
+                        .fuzzyString("string_value").levenshteinExclusive(2), // distance 1, exclusive is still fine
                 user1Oid);
+
+        searchUsersTest("with NOT levenshtein filter against string item",
+                f -> f.not().item(UserType.F_EMPLOYEE_NUMBER)
+                        .fuzzyString("User1").levenshteinInclusive(2),
+                creatorOid, modifierOid, user2Oid, user3Oid, user4Oid);
+
+        searchUsersTest("with NOT levenshtein filter against extension item",
+                f -> f.not().item(UserType.F_EXTENSION, new ItemName("string"))
+                        .fuzzyString("string_value").levenshteinExclusive(2),
+                creatorOid, modifierOid, user2Oid, user3Oid, user4Oid);
     }
 
-    @Test(expectedExceptions = SystemException.class) // the exception may change
-    public void invalidFuzzyStringSearchTest() throws SchemaException {
-        searchUsersTest("With levenshtein against no values",
-                f -> f.item(UserType.F_EMPLOYEE_NUMBER).fuzzyString().levenshtein(2, true));
+    @Test
+    public void test995InvalidFuzzyStringSearchWithNullValue() {
+        assertThatThrownBy(() -> searchUsersTest("with fuzzy filter without values",
+                f -> f.item(UserType.F_EMPLOYEE_NUMBER).fuzzyString().levenshtein(2, true)))
+                .isInstanceOf(SystemException.class) // the exception may change
+                .hasMessage("Filter 'levenshtein: employeeNumber, ' should contain exactly one value, but it contains none.");
+    }
+
+    @Test
+    public void test996InvalidFuzzyStringSearchWithMultipleValues() {
+        assertThatThrownBy(() -> searchUsersTest("with fuzzy filter with multiple values",
+                f -> f.item(UserType.F_EMPLOYEE_NUMBER).fuzzyString("first", "second").levenshtein(2, true)))
+                .isInstanceOf(SystemException.class) // the exception may change
+                .hasMessageMatching("Filter 'levenshtein: employeeNumber, .*' should contain at most one value, but it has 2 of them\\.");
     }
     // endregion
 }

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/ItemValueFilterProcessor.java

@@ -115,7 +115,8 @@ protected <T> Predicate createBinaryCondition(
     protected Predicate fuzzyStringPredicate(
             FuzzyStringMatchFilter<?> filter, Expression<?> path, ValueFilterValues<?, ?> values)
             throws QueryException {
-        return context.processFuzzyFilter(filter, path, values);
+        return predicateWithNotTreated(path,
+                context.processFuzzyFilter(filter, path, values));
     }
 
     /**