repo-sqale: using EXISTS subquery instead of JOIN for path resolution

This is a refactoring before EXISTS filter implementation trying to
unify item path resolution for both ValueFilter and ExistsFilter.

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/filtering/ExtensionItemFilterProcessor.java

@@ -44,7 +44,7 @@
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
 import com.evolveum.midpoint.repo.sqlbase.filtering.ValueFilterValues;
 import com.evolveum.midpoint.repo.sqlbase.filtering.item.FilterOperation;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
 import com.evolveum.midpoint.util.DOMUtil;
 import com.evolveum.midpoint.util.QNameUtil;
@@ -54,7 +54,7 @@
  * Filter processor for extension items stored in JSONB.
  * This takes care of any supported type, scalar or array, and handles any operation.
  */
-public class ExtensionItemFilterProcessor extends ItemFilterProcessor<ValueFilter<?, ?>> {
+public class ExtensionItemFilterProcessor extends ItemValueFilterProcessor<ValueFilter<?, ?>> {
 
     // QName.toString produces different results, QNameUtil must be used here:
     public static final String STRING_TYPE = QNameUtil.qNameToUri(DOMUtil.XSD_STRING);

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/filtering/RefItemFilterProcessor.java

@@ -22,7 +22,7 @@
 import com.evolveum.midpoint.repo.sqale.qmodel.common.QUri;
 import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.UuidPath;
 
@@ -31,7 +31,7 @@
  * OID is represented by UUID column, type by ID (see {@link MObjectType} and relation
  * by Integer (foreign key) to {@link QUri}.
  */
-public class RefItemFilterProcessor extends ItemFilterProcessor<RefFilter> {
+public class RefItemFilterProcessor extends ItemValueFilterProcessor<RefFilter> {
 
     // only oidPath is strictly not-null, but then the filter better not ask for type or relation
     private final UuidPath oidPath;

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/filtering/RefTableItemFilterProcessor.java

@@ -13,7 +13,7 @@
 import com.evolveum.midpoint.repo.sqale.qmodel.ref.QReference;
 import com.evolveum.midpoint.repo.sqale.qmodel.ref.QReferenceMapping;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
 
 /**
@@ -27,7 +27,7 @@
  */
 public class RefTableItemFilterProcessor<Q extends QReference<R, OR>, R extends MReference,
         OQ extends FlexibleRelationalPathBase<OR>, OR>
-        extends ItemFilterProcessor<RefFilter> {
+        extends ItemValueFilterProcessor<RefFilter> {
 
     private final SqlQueryContext<?, OQ, OR> context;
     private final QReferenceMapping<Q, R, OQ, OR> referenceMapping;

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/mapping/ContainerTableRelationResolver.java

@@ -9,6 +9,7 @@
 import java.util.function.BiFunction;
 
 import com.querydsl.core.types.Predicate;
+import com.querydsl.sql.SQLQuery;
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.prism.Containerable;
@@ -55,10 +56,12 @@ public ContainerTableRelationResolver(
      */
     @Override
     public ResolutionResult resolve(SqlQueryContext<?, Q, R> context) {
-        SqlQueryContext<TS, TQ, TR> joinContext =
-                context.leftJoin(targetMapping.queryType(), joinPredicate);
+        SqlQueryContext<TS, TQ, TR> subcontext =
+                context.subquery(targetMapping.queryType());
+        SQLQuery<?> subquery = subcontext.sqlQuery();
+        subquery.where(joinPredicate.apply(context.path(), subcontext.path()));
 
-        return new ResolutionResult(joinContext, joinContext.mapping());
+        return new ResolutionResult(subcontext, subcontext.mapping(), true);
     }
 
     @Override

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/mapping/ExtensionItemSqlMapper.java

@@ -12,15 +12,15 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import com.evolveum.midpoint.prism.query.ObjectFilter;
+import com.evolveum.midpoint.prism.query.ValueFilter;
 import com.evolveum.midpoint.repo.sqale.delta.ItemDeltaProcessor;
 import com.evolveum.midpoint.repo.sqale.delta.item.ExtensionItemDeltaProcessor;
 import com.evolveum.midpoint.repo.sqale.filtering.ExtensionItemFilterProcessor;
 import com.evolveum.midpoint.repo.sqale.jsonb.JsonbPath;
 import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
 import com.evolveum.midpoint.repo.sqale.update.SqaleUpdateContext;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
 
 /**
@@ -57,10 +57,10 @@ public ExtensionItemSqlMapper(
     }
 
     @Override
-    public @Nullable <T extends ObjectFilter> ItemFilterProcessor<T> createFilterProcessor(
+    public @Nullable <T extends ValueFilter<?, ?>> ItemValueFilterProcessor<T> createFilterProcessor(
             SqlQueryContext<?, ?, ?> sqlQueryContext) {
         //noinspection unchecked
-        return (ItemFilterProcessor<T>) new ExtensionItemFilterProcessor(
+        return (ItemValueFilterProcessor<T>) new ExtensionItemFilterProcessor(
                 sqlQueryContext,
                 (Function<FlexibleRelationalPathBase<?>, JsonbPath>) rootToExtensionPath,
                 holderType);

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/mapping/SqaleItemSqlMapper.java

@@ -17,7 +17,7 @@
 import com.evolveum.midpoint.repo.sqale.delta.ItemDeltaValueProcessor;
 import com.evolveum.midpoint.repo.sqale.update.SqaleUpdateContext;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.mapping.DefaultItemSqlMapper;
 import com.evolveum.midpoint.repo.sqlbase.mapping.ItemRelationResolver;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
@@ -40,15 +40,15 @@ public class SqaleItemSqlMapper<S, Q extends FlexibleRelationalPathBase<R>, R>
     Function<SqaleUpdateContext<S, Q, R>, ItemDeltaValueProcessor<?>> deltaProcessorFactory;
 
     public <P extends Path<?>> SqaleItemSqlMapper(
-            @NotNull Function<SqlQueryContext<S, Q, R>, ItemFilterProcessor<?>> filterProcessorFactory,
+            @NotNull Function<SqlQueryContext<S, Q, R>, ItemValueFilterProcessor<?>> filterProcessorFactory,
             @NotNull Function<SqaleUpdateContext<S, Q, R>, ItemDeltaValueProcessor<?>> deltaProcessorFactory,
             @Nullable Function<Q, P> primaryItemMapping) {
         super(filterProcessorFactory, primaryItemMapping);
         this.deltaProcessorFactory = Objects.requireNonNull(deltaProcessorFactory);
     }
 
     public SqaleItemSqlMapper(
-            @NotNull Function<SqlQueryContext<S, Q, R>, ItemFilterProcessor<?>> filterProcessorFactory,
+            @NotNull Function<SqlQueryContext<S, Q, R>, ItemValueFilterProcessor<?>> filterProcessorFactory,
             @NotNull Function<SqaleUpdateContext<S, Q, R>, ItemDeltaValueProcessor<?>> deltaProcessorFactory) {
         super(filterProcessorFactory);
         this.deltaProcessorFactory = Objects.requireNonNull(deltaProcessorFactory);

repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/audit/mapping/AuditCustomColumnItemFilterProcessor.java

@@ -16,7 +16,7 @@
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
 import com.evolveum.midpoint.repo.sqlbase.filtering.ValueFilterValues;
 import com.evolveum.midpoint.repo.sqlbase.filtering.item.FilterOperation;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.mapping.DefaultItemSqlMapper;
 import com.evolveum.midpoint.repo.sqlbase.mapping.ItemSqlMapper;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
@@ -35,7 +35,7 @@
  * with the same item filter processors used for non-extension columns (only the mapping
  * registration would be dynamic, which is not a big deal).
  */
-public class AuditCustomColumnItemFilterProcessor extends ItemFilterProcessor<
+public class AuditCustomColumnItemFilterProcessor extends ItemValueFilterProcessor<
         PropertyValueFilter<AuditEventRecordCustomColumnPropertyType>> {
 
     /**

repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/audit/mapping/AuditRefItemFilterProcessor.java

@@ -18,7 +18,7 @@
 import com.evolveum.midpoint.prism.query.RefFilter;
 import com.evolveum.midpoint.repo.sql.data.common.other.RObjectType;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.mapping.DefaultItemSqlMapper;
 import com.evolveum.midpoint.repo.sqlbase.mapping.ItemSqlMapper;
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
@@ -28,7 +28,7 @@
  * Filter processor for a reference attribute paths of audit-related types.
  * In audit the references have OID and (mostly, but optionally) type, but never relation.
  */
-public class AuditRefItemFilterProcessor extends ItemFilterProcessor<RefFilter> {
+public class AuditRefItemFilterProcessor extends ItemValueFilterProcessor<RefFilter> {
 
     /**
      * Returns the mapper function creating the ref-filter processor from context.

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/SqlQueryContext.java

@@ -127,7 +127,13 @@ public <T extends FlexibleRelationalPathBase<?>> T root(Class<T> rootType) {
      */
     public void processFilter(ObjectFilter filter) throws RepositoryException {
         if (filter != null) {
-            sqlQuery.where(process(filter));
+            Predicate predicate = process(filter);
+            try {
+                sqlQuery.where(predicate);
+            } catch (IllegalArgumentException e) {
+                throw new RepositoryException("Query construction problem, current query: "
+                        + sqlQuery + "\n  Predicate: " + predicate, e);
+            }
         }
     }
 
@@ -166,6 +172,11 @@ public Predicate process(@NotNull ObjectFilter filter) throws RepositoryExceptio
         }
     }
 
+    // TODO EXPLAIN
+    public Predicate transform(Predicate predicate) {
+        return predicate;
+    }
+
     /**
      * This takes care of {@link ObjectPaging} which includes ordering.
      */

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/FilterProcessor.java

@@ -11,7 +11,7 @@
 import com.evolveum.midpoint.prism.query.ObjectFilter;
 import com.evolveum.midpoint.repo.sqlbase.RepositoryException;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 
 /**
  * Filter processor is very abstract thing that takes the filter and returns the SQL predicate.
@@ -25,7 +25,7 @@
  * whether to resolve logical operations or delegate to other specialized filter.
  * *Complex path resolution* (which may add JOINs) belongs here, see {@link ValueFilterProcessor}.
  *
- * * {@link ItemFilterProcessor}s for a single Prism item (not necessarily one SQL column).
+ * * {@link ItemValueFilterProcessor}s for a single Prism item (not necessarily one SQL column).
  * These *process only single/final path component and use the value of the filter*.
  * While JOINs are typically only used here it is possible that multi-value attributes stored
  * in detail tables can generate another JOIN in this step too.

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/ValueFilterProcessor.java

@@ -12,12 +12,11 @@
 
 import com.evolveum.midpoint.prism.path.ItemName;
 import com.evolveum.midpoint.prism.path.ItemPath;
-import com.evolveum.midpoint.prism.query.ObjectFilter;
 import com.evolveum.midpoint.prism.query.ValueFilter;
 import com.evolveum.midpoint.repo.sqlbase.QueryException;
 import com.evolveum.midpoint.repo.sqlbase.RepositoryException;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
-import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
+import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemValueFilterProcessor;
 import com.evolveum.midpoint.repo.sqlbase.mapping.ItemRelationResolver;
 import com.evolveum.midpoint.repo.sqlbase.mapping.QueryModelMapping;
 
@@ -30,14 +29,15 @@
  * a filter which it processes.
  * During multi-part item path it creates necessary JOINs or subqueries creating new
  * {@link #context} and {@link #mapping} instances in the process.
- * Finally, it then delegates to the right {@link ItemFilterProcessor} to process the values
+ * Finally, it then delegates to the right {@link ItemValueFilterProcessor} to process the values
  * and construct SQL conditions.
  */
 public class ValueFilterProcessor implements FilterProcessor<ValueFilter<?, ?>> {
 
     /** Query context and mapping is not final as it can change during complex path resolution. */
     private SqlQueryContext<?, ?, ?> context;
     private QueryModelMapping<?, ?, ?> mapping;
+    private boolean transformPredicateToExists;
 
     public ValueFilterProcessor(SqlQueryContext<?, ?, ?> context) {
         this.context = context;
@@ -53,14 +53,21 @@ public Predicate process(ValueFilter<?, ?> filter) throws RepositoryException {
         }
 
         QName itemName = resolvePath(filter.getPath());
-        ItemFilterProcessor<ObjectFilter> filterProcessor =
+        ItemValueFilterProcessor<ValueFilter<?, ?>> filterProcessor =
                 mapping.itemMapper(itemName)
                         .createFilterProcessor(context);
         if (filterProcessor == null) {
             throw new QueryException("Filtering on " + filter.getPath() + " is not supported.");
             // this should not even happen, we can't even create a Query that would cause this
         }
-        return filterProcessor.process(filter);
+
+        Predicate predicate = filterProcessor.process(filter);
+        if (transformPredicateToExists) {
+            context.sqlQuery().where(predicate);
+            return context.sqlQuery().exists();
+        } else {
+            return predicate;
+        }
     }
 
     /**
@@ -84,6 +91,13 @@ private QName resolvePath(ItemPath path) throws QueryException {
             ItemRelationResolver.ResolutionResult resolution = resolver.resolve(context);
             context = resolution.context;
             mapping = resolution.mapping;
+            // If set, we don't want to reset it by e.g. another nested container.
+            // TODO: if multiple tables are crossed, only first EXISTS should be enough,
+            //  the rest can be joined to the subquery.
+            //  Nested exists must be treated too, this indicates some need for "context"
+            //  for each resolved component of the path? Or Value nested value filter processor
+            //  for the yet unresolved "tail" of the path?
+            transformPredicateToExists |= resolution.subquery;
         }
         return path.asSingleName();
     }

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/DetailTableItemFilterProcessor.java

@@ -11,8 +11,8 @@
 import com.querydsl.core.types.Predicate;
 import org.jetbrains.annotations.NotNull;
 
-import com.evolveum.midpoint.prism.query.ObjectFilter;
 import com.evolveum.midpoint.prism.query.PropertyValueFilter;
+import com.evolveum.midpoint.prism.query.ValueFilter;
 import com.evolveum.midpoint.repo.sqlbase.QueryException;
 import com.evolveum.midpoint.repo.sqlbase.RepositoryException;
 import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
@@ -24,7 +24,7 @@
 /**
  * Filter processor for a an attribute path (Prism item) that is stored in detail table.
  * Mapper using this processor defines how to get to the actual column on the detail table
- * and also takes the actual {@link ItemSqlMapper} producing the right type of {@link ItemFilterProcessor}.
+ * and also takes the actual {@link ItemSqlMapper} producing the right type of {@link ItemValueFilterProcessor}.
  *
  * @param <S> schema type for the owner of the detail table mapping
  * @param <Q> query type (entity path) from which we traverse to the detail table (owner)
@@ -33,7 +33,7 @@
  */
 public class DetailTableItemFilterProcessor
         <S, Q extends FlexibleRelationalPathBase<?>, DQ extends FlexibleRelationalPathBase<DR>, DR>
-        extends ItemFilterProcessor<PropertyValueFilter<String>> {
+        extends ItemValueFilterProcessor<PropertyValueFilter<String>> {
 
     /**
      * Creates composition mapper that defines:
@@ -91,7 +91,7 @@ public Predicate process(PropertyValueFilter<String> filter) throws RepositoryEx
         SqlQueryContext<?, DQ, DR> joinContext =
                 context.leftJoin(detailQueryType, joinOnPredicate);
 
-        FilterProcessor<ObjectFilter> filterProcessor =
+        FilterProcessor<ValueFilter<?, ?>> filterProcessor =
                 nestedItemMapper.createFilterProcessor(joinContext);
         if (filterProcessor == null) {
             throw new QueryException("Filtering on " + filter.getPath() + " is not supported.");

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/ItemFilterProcessor.java → repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/ItemValueFilterProcessor.java

@@ -30,12 +30,12 @@
  *
  * See {@link ValueFilterProcessor} for details how complex paths are resolved to its last part.
  */
-public abstract class ItemFilterProcessor<O extends ObjectFilter>
+public abstract class ItemValueFilterProcessor<O extends ValueFilter<?, ?>>
         implements FilterProcessor<O> {
 
     protected final SqlQueryContext<?, ?, ?> context;
 
-    protected ItemFilterProcessor(SqlQueryContext<?, ?, ?> context) {
+    protected ItemValueFilterProcessor(SqlQueryContext<?, ?, ?> context) {
         this.context = context;
     }
 

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/PolyStringItemFilterProcessor.java

@@ -29,7 +29,7 @@
  * Sorting is always executed by {@code *_orig} column.
  */
 public class PolyStringItemFilterProcessor
-        extends ItemFilterProcessor<PropertyValueFilter<PolyString>> {
+        extends ItemValueFilterProcessor<PropertyValueFilter<PolyString>> {
 
     public static final String STRICT = PrismConstants.POLY_STRING_STRICT_MATCHING_RULE_NAME.getLocalPart();
     public static final String ORIG = PrismConstants.POLY_STRING_ORIG_MATCHING_RULE_NAME.getLocalPart();

repo/repo-sqlbase/src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/SinglePathItemFilterProcessor.java

@@ -23,7 +23,7 @@
  * @param <P> type of the Querydsl path
  */
 public abstract class SinglePathItemFilterProcessor<T, P extends Path<?>>
-        extends ItemFilterProcessor<PropertyValueFilter<T>> {
+        extends ItemValueFilterProcessor<PropertyValueFilter<T>> {
 
     protected final P path;