repo-sqale: introduced mechanism for enriching containers (CaseWI)

Not implemented for other containers that might need it, but it should
be similar (albeit more complex for AccCertCaseWI).
This fixes TestStrings when duplicate CID problem is out of the way.

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleObjectLoader.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.repo.sqale;
+
+import java.util.Collection;
+import java.util.UUID;
+
+import org.jetbrains.annotations.NotNull;
+
+import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
+import com.evolveum.midpoint.schema.GetOperationOptions;
+import com.evolveum.midpoint.schema.SelectorOptions;
+import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
+
+/**
+ * Description of internal read-by-OID operation in the context of existing JDBC session.
+ * This is needed for enriching of some containers that need to read their owner objects.
+ */
+public interface SqaleObjectLoader {
+
+    <S extends ObjectType> S readByOid(
+            @NotNull JdbcSession jdbcSession,
+            @NotNull Class<S> schemaType,
+            @NotNull UUID oid,
+            Collection<SelectorOptions<GetOperationOptions>> options)
+            throws SchemaException, ObjectNotFoundException;
+}

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleQueryContext.java

@@ -6,12 +6,15 @@
  */
 package com.evolveum.midpoint.repo.sqale;
 
+import java.util.Collection;
+import java.util.UUID;
 import javax.xml.namespace.QName;
 
 import com.querydsl.core.types.Predicate;
 import com.querydsl.sql.SQLQuery;
 import org.jetbrains.annotations.NotNull;
 
+import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.query.*;
 import com.evolveum.midpoint.repo.sqale.filtering.ExistsFilterProcessor;
 import com.evolveum.midpoint.repo.sqale.filtering.InOidFilterProcessor;
@@ -24,15 +27,29 @@
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
 import com.evolveum.midpoint.repo.sqlbase.mapping.QueryTableMapping;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
+import com.evolveum.midpoint.schema.GetOperationOptions;
+import com.evolveum.midpoint.schema.SelectorOptions;
+import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.util.exception.SystemException;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 
 public class SqaleQueryContext<S, Q extends FlexibleRelationalPathBase<R>, R>
         extends SqlQueryContext<S, Q, R> {
 
     private boolean containsOrgFilter = false;
+    private final SqaleObjectLoader objectLoader;
 
     public static <S, Q extends FlexibleRelationalPathBase<R>, R> SqaleQueryContext<S, Q, R> from(
             Class<S> schemaType,
             SqaleRepoContext sqlRepoContext) {
+        return from(schemaType, sqlRepoContext, null);
+    }
+
+    public static <S, Q extends FlexibleRelationalPathBase<R>, R> SqaleQueryContext<S, Q, R> from(
+            Class<S> schemaType,
+            SqaleRepoContext sqlRepoContext,
+            SqaleObjectLoader objectLoader) {
 
         SqaleTableMapping<S, Q, R> rootMapping = sqlRepoContext.getMappingBySchemaType(schemaType);
         Q rootPath = rootMapping.defaultAlias();
@@ -42,15 +59,17 @@ public static <S, Q extends FlexibleRelationalPathBase<R>, R> SqaleQueryContext<
         query.getMetadata().setValidate(true);
 
         return new SqaleQueryContext<>(
-                rootPath, rootMapping, sqlRepoContext, query);
+                rootPath, rootMapping, sqlRepoContext, query, objectLoader);
     }
 
     private SqaleQueryContext(
             Q entityPath,
             SqaleTableMapping<S, Q, R> mapping,
             SqaleRepoContext sqlRepoContext,
-            SQLQuery<?> query) {
+            SQLQuery<?> query,
+            SqaleObjectLoader objectLoader) {
         super(entityPath, mapping, sqlRepoContext, query);
+        this.objectLoader = objectLoader;
     }
 
     private SqaleQueryContext(
@@ -59,6 +78,7 @@ private SqaleQueryContext(
             SqaleQueryContext<?, ?, ?> parentContext,
             SQLQuery<?> sqlQuery) {
         super(entityPath, mapping, parentContext, sqlQuery);
+        this.objectLoader = parentContext.objectLoader;
     }
 
     @Override
@@ -119,6 +139,18 @@ public void markContainsOrgFilter() {
                 query);
     }
 
+    public <T extends ObjectType> PrismObject<T> loadObject(
+            JdbcSession jdbcSession, Class<T> objectType, UUID oid,
+            Collection<SelectorOptions<GetOperationOptions>> options) {
+        try {
+            //noinspection unchecked
+            return (PrismObject<T>) objectLoader.readByOid(jdbcSession, objectType, oid, options)
+                    .asPrismObject();
+        } catch (SchemaException | ObjectNotFoundException e) {
+            throw new SystemException(e);
+        }
+    }
+
     @Override
     public SqaleQueryContext<?, ?, ?> parentContext() {
         return (SqaleQueryContext<?, ?, ?>) super.parentContext();

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryService.java

@@ -1110,9 +1110,7 @@ public <T extends Containerable> SearchResultList<T> searchContainers(
                 return new SearchResultList<>();
             }
 
-            SqaleQueryContext<T, FlexibleRelationalPathBase<Object>, Object> queryContext =
-                    SqaleQueryContext.from(type, sqlRepoContext);
-            return sqlQueryExecutor.list(queryContext, query, options);
+            return executeSearchContainers(type, query, options);
         } catch (RepositoryException | RuntimeException e) {
             throw handledGeneralException(e, operationResult);
         } catch (Throwable t) {
@@ -1123,6 +1121,20 @@ public <T extends Containerable> SearchResultList<T> searchContainers(
         }
     }
 
+    private <T extends Containerable> SearchResultList<T> executeSearchContainers(Class<T> type,
+            ObjectQuery query, Collection<SelectorOptions<GetOperationOptions>> options)
+            throws RepositoryException, SchemaException {
+
+        long opHandle = registerOperationStart(OP_SEARCH_CONTAINERS, type);
+        try {
+            SqaleQueryContext<T, FlexibleRelationalPathBase<Object>, Object> queryContext =
+                    SqaleQueryContext.from(type, sqlRepoContext, this::readByOid);
+            return sqlQueryExecutor.list(queryContext, query, options);
+        } finally {
+            registerOperationFinish(opHandle, 1);
+        }
+    }
+
     private <T> void logSearchInputParameters(Class<T> type, ObjectQuery query, String operation) {
         ObjectPaging paging = query != null ? query.getPaging() : null;
         logger.debug(

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/audit/SqaleAuditServiceFactory.java

@@ -6,12 +6,14 @@
  */
 package com.evolveum.midpoint.repo.sqale.audit;
 
-import static com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration.PROPERTY_DATASOURCE;
-import static com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration.PROPERTY_JDBC_URL;
+import static com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration.*;
 
+import java.sql.Types;
 import java.util.List;
 import javax.sql.DataSource;
 
+import com.google.common.base.Strings;
+import com.querydsl.sql.ColumnMetadata;
 import org.apache.commons.configuration2.BaseHierarchicalConfiguration;
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.configuration2.HierarchicalConfiguration;
@@ -24,13 +26,18 @@
 import com.evolveum.midpoint.repo.api.SqlPerformanceMonitorsCollection;
 import com.evolveum.midpoint.repo.sqale.SqaleRepoContext;
 import com.evolveum.midpoint.repo.sqale.SqaleRepositoryBeanConfig;
+import com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration;
 import com.evolveum.midpoint.repo.sqale.audit.qmodel.QAuditDeltaMapping;
+import com.evolveum.midpoint.repo.sqale.audit.qmodel.QAuditEventRecord;
 import com.evolveum.midpoint.repo.sqale.audit.qmodel.QAuditEventRecordMapping;
 import com.evolveum.midpoint.repo.sqale.audit.qmodel.QAuditRefValueMapping;
 import com.evolveum.midpoint.repo.sqlbase.DataSourceFactory;
 import com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration;
+import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
+import com.evolveum.midpoint.repo.sqlbase.SqlTableMetadata;
 import com.evolveum.midpoint.repo.sqlbase.mapping.QueryModelMappingRegistry;
 import com.evolveum.midpoint.schema.SchemaService;
+import com.evolveum.midpoint.util.exception.SystemException;
 import com.evolveum.midpoint.util.logging.Trace;
 import com.evolveum.midpoint.util.logging.TraceManager;
 import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
@@ -50,15 +57,15 @@ public class SqaleAuditServiceFactory implements AuditServiceFactory {
     private static final String CONF_AUDIT_SERVICE_COLUMN_NAME = "columnName";
     private static final String CONF_AUDIT_SERVICE_EVENT_RECORD_PROPERTY_NAME = "eventRecordPropertyName";
 
-    private final com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration sqaleRepositoryConfiguration;
+    private final SqaleRepositoryConfiguration sqaleRepositoryConfiguration;
     private final SchemaService schemaService;
     private final DataSource repositoryDataSource;
     private final SqlPerformanceMonitorsCollection sqlPerformanceMonitorsCollection;
 
     private SqaleAuditService auditService;
 
     public SqaleAuditServiceFactory(
-            com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration sqaleRepositoryConfiguration,
+            SqaleRepositoryConfiguration sqaleRepositoryConfiguration,
             SchemaService schemaService,
             DataSource repositoryDataSource,
             SqlPerformanceMonitorsCollection sqlPerformanceMonitorsCollection) {
@@ -97,7 +104,7 @@ private SqaleRepoContext createSqaleRepoContext(Configuration configuration)
         }
 
         LOGGER.info("Configuring SQL audit service to use a different datasource");
-        com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration config = new com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration(configuration);
+        SqaleRepositoryConfiguration config = new SqaleRepositoryConfiguration(configuration);
 
         DataSourceFactory dataSourceFactory = new DataSourceFactory(config);
         DataSource dataSource = dataSourceFactory.createDataSource();
@@ -129,9 +136,8 @@ private void initCustomColumns(
                         .configurationsAt(CONF_AUDIT_SERVICE_COLUMNS);
 
         // here we use config from context, it can be main repository configuration
-        com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration repoConfig =
-                (com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration) sqlRepoContext.getJdbcRepositoryConfiguration();
-        /*
+        SqaleRepositoryConfiguration repoConfig =
+                (SqaleRepositoryConfiguration) sqlRepoContext.getJdbcRepositoryConfiguration();
         boolean createMissing = repoConfig.isCreateMissingCustomColumns()
                 // but we'll consider the flag also on audit configuration, just in case
                 || configuration.getBoolean(PROPERTY_CREATE_MISSING_CUSTOM_COLUMNS, false);
@@ -155,8 +161,7 @@ private void initCustomColumns(
             QAuditEventRecordMapping.get().addExtensionColumn(propertyName, columnMetadata);
             if (tableMetadata != null && tableMetadata.get(columnName) == null) {
                 // Fails on SQL Server with snapshot transaction, so different isolation is used.
-                try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession()
-                        .startTransaction(Connection.TRANSACTION_READ_COMMITTED)) {
+                try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
                     jdbcSession.addColumn(QAuditEventRecord.TABLE_NAME,
                             ColumnMetadata.named(columnName).ofType(Types.VARCHAR).withSize(255));
                     jdbcSession.commit();
@@ -177,7 +182,6 @@ private String getStringFromConfig(Configuration config, String key) {
 
         return value;
 
-         */
     }
 
     @Override

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/mapping/SqaleTableMapping.java

@@ -248,24 +248,10 @@ protected ItemSqlMapper<Q, R> multiUriMapper(
                         ctx.repositoryContext()::processCacheableUri));
     }
 
-    /**
-     * Implemented for searchable containers that do not use fullObject for their recreation.
-     */
     @Override
     public S toSchemaObject(R row) throws SchemaException {
-        throw new UnsupportedOperationException("Use toSchemaObject(Tuple,...)");
-    }
-
-    /**
-     * Transforms row Tuple containing {@link R} under entity path and extension columns.
-     */
-    @Override
-    public S toSchemaObject(Tuple tuple, Q entityPath,
-            Collection<SelectorOptions<GetOperationOptions>> options)
-            throws SchemaException {
-        S schemaObject = toSchemaObject(tuple.get(entityPath));
-        processExtensionColumns(schemaObject, tuple, entityPath);
-        return schemaObject;
+        throw new UnsupportedOperationException(
+                "Not implemented for " + getClass() + ". Perhaps use toSchemaObject(Tuple,...)?");
     }
 
     // TODO reconsider, if not necessary in 2023 DELETE (originally meant for ext item per column,

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/accesscert/QAccessCertificationWorkItemMapping.java

@@ -82,6 +82,7 @@ private QAccessCertificationWorkItemMapping(@NotNull SqaleRepoContext repository
     @Override
     public AccessCertificationWorkItemType toSchemaObject(MAccessCertificationWorkItem row) {
         AccessCertificationWorkItemType acwi = new AccessCertificationWorkItemType(prismContext())
+                .id(row.cid)
                 .closeTimestamp(asXMLGregorianCalendar(row.closeTimestamp))
                 .iteration(row.campaignIteration)
                 .outputChangeTimestamp(asXMLGregorianCalendar(row.outputChangeTimestamp))

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/cases/workitem/QCaseWorkItemMapping.java

@@ -9,20 +9,28 @@
 import static com.evolveum.midpoint.util.MiscUtil.asXMLGregorianCalendar;
 import static com.evolveum.midpoint.xml.ns._public.common.common_3.CaseWorkItemType.*;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
+import java.util.UUID;
 
 import org.jetbrains.annotations.NotNull;
 
-import com.evolveum.midpoint.prism.PrismConstants;
+import com.evolveum.midpoint.prism.*;
+import com.evolveum.midpoint.repo.sqale.SqaleQueryContext;
 import com.evolveum.midpoint.repo.sqale.SqaleRepoContext;
 import com.evolveum.midpoint.repo.sqale.qmodel.cases.MCase;
 import com.evolveum.midpoint.repo.sqale.qmodel.cases.QCaseMapping;
 import com.evolveum.midpoint.repo.sqale.qmodel.common.QContainerMapping;
 import com.evolveum.midpoint.repo.sqale.qmodel.focus.QUserMapping;
 import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
+import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
+import com.evolveum.midpoint.repo.sqlbase.mapping.ResultListRowTransformer;
 import com.evolveum.midpoint.repo.sqlbase.mapping.TableRelationResolver;
 import com.evolveum.midpoint.util.MiscUtil;
+import com.evolveum.midpoint.util.exception.SystemException;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.AbstractWorkItemOutputType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.CaseType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.CaseWorkItemType;
 
 /**
@@ -90,6 +98,7 @@ private QCaseWorkItemMapping(@NotNull SqaleRepoContext repositoryContext) {
     @Override
     public CaseWorkItemType toSchemaObject(MCaseWorkItem row) {
         CaseWorkItemType cwi = new CaseWorkItemType(prismContext())
+                .id(row.cid)
                 .closeTimestamp(asXMLGregorianCalendar(row.closeTimestamp))
                 .createTimestamp(asXMLGregorianCalendar(row.createTimestamp))
                 .deadline(asXMLGregorianCalendar(row.deadline))
@@ -106,6 +115,33 @@ public CaseWorkItemType toSchemaObject(MCaseWorkItem row) {
         return cwi;
     }
 
+    @Override
+    public ResultListRowTransformer<CaseWorkItemType, QCaseWorkItem, MCaseWorkItem> createRowTransformer(
+            SqlQueryContext<CaseWorkItemType, QCaseWorkItem, MCaseWorkItem> sqlQueryContext, JdbcSession jdbcSession) {
+        Map<UUID, PrismObject<CaseType>> casesCache = new HashMap<>();
+
+        return (tuple, entityPath, options) -> {
+            MCaseWorkItem row = Objects.requireNonNull(tuple.get(entityPath));
+            UUID caseOid = row.ownerOid;
+            PrismObject<CaseType> aCase = casesCache.get(caseOid);
+            if (aCase == null) {
+                aCase = ((SqaleQueryContext<CaseWorkItemType, QCaseWorkItem, MCaseWorkItem>) sqlQueryContext)
+                        .loadObject(jdbcSession, CaseType.class, caseOid, options);
+                casesCache.put(caseOid, aCase);
+            }
+
+            PrismContainer<Containerable> workItemContainer = aCase.findContainer(CaseType.F_WORK_ITEM);
+            if (workItemContainer == null) {
+                throw new SystemException("Case " + aCase + " has no work items even if it should have " + tuple);
+            }
+            PrismContainerValue<?> workItemPcv = workItemContainer.findValue(row.cid);
+            if (workItemPcv == null) {
+                throw new SystemException("Case " + aCase + " has no work item with ID " + row.cid);
+            }
+            return (CaseWorkItemType) workItemPcv.asContainerable();
+        };
+    }
+
     @Override
     protected QCaseWorkItem newAliasInstance(String alias) {
         return new QCaseWorkItem(alias);