native-repo: added identity skipping/resolving, toSchemaObject review

Explicit toSchemaObjectWithResolvedNames will be introduced, WIP.

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryService.java

@@ -221,7 +221,7 @@ private <S extends ObjectType> S readByOid(
             throw new ObjectNotFoundException(schemaType, oid.toString());
         }
 
-        return rootMapping.toSchemaObject(result, root, options, jdbcSession, false);
+        return rootMapping.toSchemaObjectWithResolvedNames(result, root, options, jdbcSession, false);
     }
 
     @Override
@@ -704,7 +704,7 @@ RootUpdateContext<S, Q, R> prepareUpdateContext(
             throw new ObjectNotFoundException(schemaType, oid.toString());
         }
 
-        S object = rootMapping.toSchemaObject(
+        S object = rootMapping.toSchemaObjectWithResolvedNames(
                 result, entityPath, getOptions, jdbcSession, RepoModifyOptions.isForceReindex(options));
 
         R rootRow = rootMapping.newRowObject();

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/mapping/SqaleTableMapping.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2010-2021 Evolveum and contributors
+ * Copyright (C) 2010-2022 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
@@ -291,10 +291,6 @@ public S toSchemaObject(Tuple tuple, Q entityPath,
         return schemaObject;
     }
 
-    // TODO reconsider, if not necessary in 2023 DELETE (originally meant for ext item per column,
-    //  but can this be used for adding index-only exts to schema object even from JSON?)
-
-    @SuppressWarnings("unused")
     protected void processExtensionColumns(S schemaObject, Tuple tuple, Q entityPath) {
         // empty by default, can be overridden
     }
@@ -511,15 +507,35 @@ protected Collection<? extends QName> fullObjectItemsToSkip() {
         return Collections.emptyList();
     }
 
+    // TODO: this resolves the names after toSchemaObject(3 params), but...
+    //  called version of toSchemaObject() doesn't have forceFull flag, so sometimes we need to override this version.
+    //  But calling this via super...() means it can't resolve the names in refs from additional parts
+    //  that are not loaded yet (which is done after super call).
+    //  What is the recommended template for override? Wouldn't using this method with additional stuff in the middle be better?
+    //  1. call toSchemaObject(rowTuple, entityPath, options) - just like here, don't call this 5 param method via super!
+    //  2. do additional stuff, utilizing forceFull flag (and even jdbcSession if necessary)
+    //  3. call resolveReferenceNames just like in this method - at the end
+    //  Alternative:
+    //  - make this method final
+    //  - override only toSchemaObject - but add forceFull flag (do we need jdbcSession as well? so far not)
     public S toSchemaObject(
             Tuple rowTuple,
             Q entityPath,
             Collection<SelectorOptions<GetOperationOptions>> options,
             @NotNull JdbcSession jdbcSession,
             boolean forceFull) throws SchemaException {
-        S ret = toSchemaObject(rowTuple, entityPath, options);
-        ret = resolveNames(ret, jdbcSession, options);
-        return ret;
+        return toSchemaObject(rowTuple, entityPath, options);
+    }
+
+    public S toSchemaObjectWithResolvedNames(
+            Tuple rowTuple,
+            Q entityPath,
+            Collection<SelectorOptions<GetOperationOptions>> options,
+            @NotNull JdbcSession jdbcSession,
+            boolean forceFull) throws SchemaException {
+        S schemaObject = toSchemaObject(rowTuple, entityPath, options, jdbcSession, forceFull);
+        schemaObject = resolveReferenceNames(schemaObject, jdbcSession, options);
+        return schemaObject;
     }
 
     public S toSchemaObjectSafe(
@@ -529,13 +545,13 @@ public S toSchemaObjectSafe(
             @NotNull JdbcSession jdbcSession,
             boolean forceFull) {
         try {
-            return toSchemaObject(tuple, entityPath, options, jdbcSession, forceFull);
+            return toSchemaObjectWithResolvedNames(tuple, entityPath, options, jdbcSession, forceFull);
         } catch (SchemaException e) {
             throw new RepositoryMappingException(e);
         }
     }
 
-    protected <O> O resolveNames(O object, JdbcSession session, Collection<SelectorOptions<GetOperationOptions>> options) {
+    protected <O> O resolveReferenceNames(O object, JdbcSession session, Collection<SelectorOptions<GetOperationOptions>> options) {
         // TODO: Performance: This could be transaction shared object
         return ReferenceNameResolver.from(options).resolve(object, session);
     }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/accesscert/QAccessCertificationCampaignMapping.java

@@ -142,11 +142,12 @@ public void storeRelatedEntities(
         }
     }
 
+    // TODO rework to toSchemaObject
     @Override
-    public AccessCertificationCampaignType toSchemaObject(Tuple rowTuple, QAccessCertificationCampaign entityPath,
+    public AccessCertificationCampaignType toSchemaObjectWithResolvedNames(Tuple rowTuple, QAccessCertificationCampaign entityPath,
             Collection<SelectorOptions<GetOperationOptions>> options, @NotNull JdbcSession jdbcSession,
             boolean forceFull) throws SchemaException {
-        AccessCertificationCampaignType base = super.toSchemaObject(rowTuple, entityPath, options, jdbcSession, forceFull);
+        AccessCertificationCampaignType base = super.toSchemaObjectWithResolvedNames(rowTuple, entityPath, options, jdbcSession, forceFull);
         if (forceFull || shouldLoadCases(options)) {
             loadCases(base, options, jdbcSession, forceFull);
         }
@@ -194,7 +195,7 @@ private void loadCases(AccessCertificationCampaignType base, Collection<Selector
         }
         List<Tuple> rows = query.fetch();
         for (Tuple row : rows) {
-            AccessCertificationCaseType c = casesMapping.toSchemaObject(row, qcase, options, jdbcSession, forceFull);
+            AccessCertificationCaseType c = casesMapping.toSchemaObjectWithResolvedNames(row, qcase, options, jdbcSession, forceFull);
             cases.add(c.asPrismContainerValue());
         }
     }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/accesscert/QAccessCertificationWorkItemMapping.java

@@ -193,7 +193,7 @@ public ResultListRowTransformer<AccessCertificationWorkItemType, QAccessCertific
                     throw new SystemException(e);
                 }
             }
-            resolveNames(aCase.asContainerable(), jdbcSession, options);
+            resolveReferenceNames(aCase.asContainerable(), jdbcSession, options);
 
             PrismContainer<AccessCertificationWorkItemType> container =
                     aCase.findContainer(AccessCertificationCaseType.F_WORK_ITEM);
@@ -205,7 +205,7 @@ public ResultListRowTransformer<AccessCertificationWorkItemType, QAccessCertific
                 throw new SystemException("Campaign " + owner + "has no work item with ID " + row.cid);
             }
             @NotNull AccessCertificationWorkItemType ret = value.asContainerable();
-            resolveNames(ret, jdbcSession, options);
+            resolveReferenceNames(ret, jdbcSession, options);
             return ret;
         };
     }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/focus/QFocusMapping.java

@@ -8,6 +8,7 @@
 
 import static com.evolveum.midpoint.xml.ns._public.common.common_3.FocusType.*;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
@@ -19,12 +20,14 @@
 
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.PrismProperty;
+import com.evolveum.midpoint.prism.delta.ItemDelta;
 import com.evolveum.midpoint.prism.util.PrismUtil;
 import com.evolveum.midpoint.repo.sqale.SqaleRepoContext;
 import com.evolveum.midpoint.repo.sqale.qmodel.object.QAssignmentHolderMapping;
 import com.evolveum.midpoint.repo.sqale.qmodel.ref.QObjectReferenceMapping;
 import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
 import com.evolveum.midpoint.schema.GetOperationOptions;
+import com.evolveum.midpoint.schema.SchemaService;
 import com.evolveum.midpoint.schema.SelectorOptions;
 import com.evolveum.midpoint.util.MiscUtil;
 import com.evolveum.midpoint.util.exception.SchemaException;
@@ -223,4 +226,16 @@ public void storeRelatedEntities(
             }
         }
     }
+
+    @Override
+    public Collection<SelectorOptions<GetOperationOptions>> updateGetOptions(
+            Collection<SelectorOptions<GetOperationOptions>> options,
+            @NotNull Collection<? extends ItemDelta<?, ?>> modifications) {
+        List<SelectorOptions<GetOperationOptions>> ret = new ArrayList<>(super.updateGetOptions(options, modifications));
+
+        if (modifications.stream().anyMatch(m -> F_IDENTITIES.isSubPath(m.getPath()))) {
+            ret.addAll(SchemaService.get().getOperationOptionsBuilder().item(F_IDENTITIES).retrieve().build());
+        }
+        return ret;
+    }
 }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/lookuptable/QLookupTableMapping.java

@@ -98,11 +98,12 @@ protected Collection<? extends QName> fullObjectItemsToSkip() {
         return Collections.singleton(F_ROW);
     }
 
+    // TODO rework to toSchemaObject
     @Override
-    public LookupTableType toSchemaObject(
+    public LookupTableType toSchemaObjectWithResolvedNames(
             Tuple rowTuple, QLookupTable entityPath, Collection<SelectorOptions<GetOperationOptions>> options,
             @NotNull JdbcSession session, boolean forceFull) throws SchemaException {
-        LookupTableType base = super.toSchemaObject(rowTuple, entityPath, options, session, forceFull);
+        LookupTableType base = super.toSchemaObjectWithResolvedNames(rowTuple, entityPath, options, session, forceFull);
 
         if (forceFull || SelectorOptions.hasToLoadPath(F_ROW, options)) {
             @Nullable GetOperationOptions rowOptions = findLookupTableGetOption(options);

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/object/QObjectMapping.java

@@ -177,7 +177,7 @@ public S toSchemaObjectSafe(
             @NotNull JdbcSession jdbcSession,
             boolean forceFull) {
         try {
-            return toSchemaObject(tuple, entityPath, options, jdbcSession, forceFull);
+            return toSchemaObjectWithResolvedNames(tuple, entityPath, options, jdbcSession, forceFull);
         } catch (SchemaException e) {
             try {
                 PrismObject<S> errorObject = prismContext().createObject(schemaType());

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/shadow/QShadowMapping.java

@@ -225,14 +225,7 @@ public Collection<SelectorOptions<GetOperationOptions>> updateGetOptions(
             @NotNull Collection<? extends ItemDelta<?, ?>> modifications) {
         List<SelectorOptions<GetOperationOptions>> ret = new ArrayList<>(super.updateGetOptions(options, modifications));
 
-        boolean attributes = false;
-        for (ItemDelta<?, ?> modification : modifications) {
-            if (F_ATTRIBUTES.isSubPath(modification.getPath())) {
-                attributes = true;
-                break;
-            }
-        }
-        if (attributes) {
+        if (modifications.stream().anyMatch(m -> F_ATTRIBUTES.isSubPath(m.getPath()))) {
             ret.addAll(SchemaService.get().getOperationOptionsBuilder().item(F_ATTRIBUTES).retrieve().build());
         }
         return ret;

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoIdentityDataTest.java

@@ -6,16 +6,24 @@
  */
 package com.evolveum.midpoint.repo.sqale.func;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import java.io.File;
 import java.io.IOException;
+import java.util.Collection;
 
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import com.evolveum.midpoint.prism.PrismContainerValue;
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.repo.sqale.SqaleRepoBaseTest;
+import com.evolveum.midpoint.schema.GetOperationOptions;
+import com.evolveum.midpoint.schema.SchemaService;
+import com.evolveum.midpoint.schema.SelectorOptions;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.util.exception.CommonException;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.FocusType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType;
 
 // TEMPORARY CODE
@@ -45,11 +53,28 @@ public void test100AddUserWithIdentityData()
         displayValue("user to add (XML)", prismContext.xmlSerializer().serialize(userToAdd));
 
         when("addObject is called");
-        repositoryService.addObject(userToAdd, null, result);
+        String oid = repositoryService.addObject(userToAdd, null, result);
 
         then("operation is successful");
         assertThatOperationResult(result).isSuccess();
 
-        // TODO
+        and("user can be obtained from repo, by default without identities");
+        OperationResult getResult = createOperationResult();
+        UserType user = repositoryService.getObject(UserType.class, oid, null, getResult).asObjectable();
+        assertThatOperationResult(getResult).isSuccess();
+        // container is marked incomplete and its value is empty
+        assertThat(user.asPrismObject().findContainer(FocusType.F_IDENTITIES).isIncomplete()).isTrue();
+        assertThat(((PrismContainerValue<?>) user.getIdentities().asPrismContainerValue()).isEmpty()).isTrue();
+
+        and("user can be obtained with identities using options");
+        Collection<SelectorOptions<GetOperationOptions>> getOptions = SchemaService.get()
+                .getOperationOptionsBuilder().item(FocusType.F_IDENTITIES).retrieve().build();
+        OperationResult getWithIdentitiesResult = createOperationResult();
+        UserType user2 = repositoryService.getObject(UserType.class, oid, getOptions, getWithIdentitiesResult).asObjectable();
+        assertThatOperationResult(getWithIdentitiesResult).isSuccess();
+        // TODO fix
+//        assertThat(user2.asPrismObject().findContainer(FocusType.F_IDENTITIES).isIncomplete()).isFalse();
+
+        // TODO more in-depth check of identity
     }
 }

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/mapping/QueryTableMapping.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2010-2021 Evolveum and contributors
+ * Copyright (C) 2010-2022 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
@@ -40,7 +40,7 @@
 /**
  * Common supertype for mapping items/attributes between schema (prism) classes and tables.
  * See {@link #addItemMapping(QName, ItemSqlMapper)} for details about mapping mechanism.
- * See {@link #addDetailFetchMapper(ItemName, SqlDetailFetchMapper)} for more about mapping
+ * See {@link #createRowTransformer(SqlQueryContext, JdbcSession)} for more about mapping
  * related to-many detail tables.
  *
  * The main goal of this type is to map object query conditions and ORDER BY to SQL.
@@ -174,10 +174,14 @@ protected ItemSqlMapper<Q, R> polyStringMapper(
      * this mapped entity (master).
      * One fetcher per detail type/table is registered under the related item name.
      *
+     * Used only in old-repo audit, we will let it die with that - but don't use in new stuff.
+     * Use {@link #createRowTransformer(SqlQueryContext, JdbcSession)} mechanism instead.
+     *
      * @param itemName item name from schema type that is mapped to detail table in the repository
      * @param detailFetchMapper fetcher-mapper that handles loading of details
      * @see SqlDetailFetchMapper
      */
+    @Deprecated
     public final void addDetailFetchMapper(
             ItemName itemName,
             SqlDetailFetchMapper<R, ?, ?, ?> detailFetchMapper) {
@@ -248,6 +252,9 @@ public synchronized Q defaultAlias() {
 
     /**
      * Returns collection of all registered {@link SqlDetailFetchMapper}s.
+     *
+     * Used only in old-repo audit, we will let it die with that - but don't use in new stuff.
+     * Use {@link #createRowTransformer(SqlQueryContext, JdbcSession)} mechanism instead.
      */
     @Deprecated
     public final Collection<SqlDetailFetchMapper<R, ?, ?, ?>> detailFetchMappers() {
@@ -298,8 +305,8 @@ public R newRowObject() {
      * which is OK if extension columns are used only for query and their information
      * is still contained in the object somehow else (e.g. full object LOB).
      *
-     * Alternative would be dynamically generated list of select expressions and transforming
-     * row to M object directly from {@link com.querydsl.core.Tuple}.
+     * Alternative is to dynamically generate the list of select expressions reading directly from
+     * the {@link com.querydsl.core.Tuple} - see {@link #toSchemaObject(Tuple, FlexibleRelationalPathBase, Collection)}.
      */
     public abstract S toSchemaObject(R row) throws SchemaException;
 
@@ -316,10 +323,10 @@ public S toSchemaObject(Tuple row, Q entityPath,
     }
 
     /**
-     * Similarly, transformation to midPoint objects allows for state using {@link ResultListRowTransformer}
-     * instead of is done in one-by-one manner, it is not done by
-     * the mapping (which is otherwise stateless), but the mapping creates transformer , there is also room
-     * for a stateful object
+     * Returns result transformer that by default calls
+     * {@link #toSchemaObject(Tuple, FlexibleRelationalPathBase, Collection)} for each result row.
+     * This can be overridden, see {@link ResultListRowTransformer} javadoc for details.
+     * This is useful for stateful transformers where the whole result can be pre-/post-processed as well.
      */
     public ResultListRowTransformer<S, Q, R> createRowTransformer(
             SqlQueryContext<S, Q, R> sqlQueryContext, JdbcSession jdbcSession) {
@@ -332,6 +339,11 @@ public ResultListRowTransformer<S, Q, R> createRowTransformer(
         };
     }
 
+    public Collection<SelectorOptions<GetOperationOptions>> updateGetOptions(Collection<SelectorOptions<GetOperationOptions>> options,
+            @NotNull Collection<? extends ItemDelta<?, ?>> modifications) {
+        return options;
+    }
+
     @Override
     public String toString() {
         return "QueryTableMapping{" +
@@ -341,9 +353,4 @@ public String toString() {
                 ", queryType=" + queryType() +
                 '}';
     }
-
-    public Collection<SelectorOptions<GetOperationOptions>> updateGetOptions(Collection<SelectorOptions<GetOperationOptions>> options,
-            @NotNull Collection<? extends ItemDelta<?, ?>> modifications) {
-        return options;
-    }
 }