Skip to content

Commit

Permalink
repo-sqale: added query support for ref extension items
Browse files Browse the repository at this point in the history 
This required change in ExtensionItemFilterProcessor which does not
extend from SinglePathItemFilterProcessor anymore (because it supports
only PropertyValueFilter) but from ItemFilterProcessor to allow
RefFilters too.
  • Loading branch information
@virgo47
virgo47 committed Jun 21, 2021
1 parent 4056a33 commit 5e1d82b
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 41 deletions.
116 changes: 94 additions & 22 deletions ...rc/main/java/com/evolveum/midpoint/repo/sqale/filtering/ExtensionItemFilterProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,43 +15,45 @@
import static com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemCardinality.SCALAR;
import static com.evolveum.midpoint.repo.sqlbase.filtering.item.PolyStringItemFilterProcessor.*;

import java.util.List;
import java.util.function.Function;
import javax.xml.datatype.XMLGregorianCalendar;

import com.google.common.base.Strings;
import com.querydsl.core.types.ExpressionUtils;
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.dsl.BooleanExpression;
import org.jetbrains.annotations.NotNull;

import com.evolveum.midpoint.prism.PrismConstants;
import com.evolveum.midpoint.prism.PrismPropertyDefinition;
import com.evolveum.midpoint.prism.*;
import com.evolveum.midpoint.prism.polystring.PolyString;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.prism.query.PropertyValueFilter;
import com.evolveum.midpoint.prism.query.RefFilter;
import com.evolveum.midpoint.prism.query.ValueFilter;
import com.evolveum.midpoint.repo.sqale.ExtUtils;
import com.evolveum.midpoint.repo.sqale.SqaleQueryContext;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItem;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
import com.evolveum.midpoint.repo.sqlbase.QueryException;
import com.evolveum.midpoint.repo.sqlbase.RepositoryException;
import com.evolveum.midpoint.repo.sqlbase.SqlQueryContext;
import com.evolveum.midpoint.repo.sqlbase.filtering.ValueFilterValues;
import com.evolveum.midpoint.repo.sqlbase.filtering.item.FilterOperation;
import com.evolveum.midpoint.repo.sqlbase.filtering.item.SinglePathItemFilterProcessor;
import com.evolveum.midpoint.repo.sqlbase.filtering.item.ItemFilterProcessor;
import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
import com.evolveum.midpoint.repo.sqlbase.querydsl.JsonbPath;
import com.evolveum.midpoint.util.DOMUtil;
import com.evolveum.midpoint.util.QNameUtil;
import com.evolveum.prism.xml.ns._public.types_3.ObjectReferenceType;
import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;

/**
* Filter processor for extension items stored in JSONB.
* This takes care of any supported type, scalar or array, and handles any operation.
*/
public class ExtensionItemFilterProcessor
extends SinglePathItemFilterProcessor<Object, JsonbPath> {
public class ExtensionItemFilterProcessor<T extends PrismValue>
extends ItemFilterProcessor<ValueFilter<T, ?>> {

// QName.toString produces different results, QNameUtil must be used here:
public static final String STRING_TYPE = QNameUtil.qNameToUri(DOMUtil.XSD_STRING);
Expand All @@ -65,45 +67,49 @@ public class ExtensionItemFilterProcessor
public static final String BOOLEAN_TYPE = QNameUtil.qNameToUri(DOMUtil.XSD_BOOLEAN);
public static final String DATETIME_TYPE = QNameUtil.qNameToUri(DOMUtil.XSD_DATETIME);
public static final String POLY_STRING_TYPE = QNameUtil.qNameToUri(PolyStringType.COMPLEX_TYPE);
public static final String REF_TYPE = QNameUtil.qNameToUri(ObjectReferenceType.COMPLEX_TYPE);

private final MExtItemHolderType holderType;
protected final JsonbPath path;

public <Q extends FlexibleRelationalPathBase<R>, R> ExtensionItemFilterProcessor(
SqlQueryContext<?, Q, R> context,
Function<Q, JsonbPath> rootToExtensionPath,
MExtItemHolderType holderType) {
super(context, rootToExtensionPath);
super(context);

this.path = rootToExtensionPath.apply(context.path());
this.holderType = holderType;
}

@Override
public Predicate process(PropertyValueFilter<Object> filter) throws RepositoryException {
PrismPropertyDefinition<?> definition = filter.getDefinition();
public Predicate process(ValueFilter<T, ?> filter) throws RepositoryException {
ItemDefinition<?> definition = filter.getDefinition();
MExtItem extItem = ((SqaleQueryContext<?, ?, ?>) context).repositoryContext()
.resolveExtensionItem(definition, holderType);
assert definition != null;

ValueFilterValues<?, ?> values = ValueFilterValues.from(filter);
if (definition instanceof PrismReferenceDefinition) {
return processReference(extItem, (RefFilter) filter);
}

//noinspection unchecked
PropertyValueFilter<T> propertyValueFilter = (PropertyValueFilter<T>) filter;
ValueFilterValues<?, ?> values = ValueFilterValues.from(propertyValueFilter);
// TODO where do we want tu support eq with multiple values?
FilterOperation operation = operation(filter);

if (values.isEmpty()) {
List<T> filterValues = filter.getValues();
if (filterValues == null || filterValues.isEmpty()) {
if (operation.isAnyEqualOperation()) {
// ?? is "escaped" ? operator, PG JDBC driver understands it. Alternative is to use
// function jsonb_exists but that does NOT use GIN index, only operators do!
// We have to use parenthesis with AND shovelled into the template like this.
return booleanTemplate("({0} ?? {1} AND {0} is not null)",
path, extItem.id.toString()).not();
return extItemIsNull(extItem);
} else {
throw new QueryException("Null value for other than EQUAL filter: " + filter);
}
}

if (extItem.valueType.equals(STRING_TYPE)) {
return processString(extItem, values, operation, filter);
} else if (ExtUtils.isEnumDefinition(definition)) {
} else if (ExtUtils.isEnumDefinition((PrismPropertyDefinition<?>) definition)) {
return processEnum(extItem, values, operation, filter);
} else if (extItem.valueType.equals(INT_TYPE) || extItem.valueType.equals(INTEGER_TYPE)
|| extItem.valueType.equals(LONG_TYPE) || extItem.valueType.equals(SHORT_TYPE)
Expand All @@ -115,19 +121,76 @@ public Predicate process(PropertyValueFilter<Object> filter) throws RepositoryEx
} else if (extItem.valueType.equals(DATETIME_TYPE)) {
//noinspection unchecked
PropertyValueFilter<XMLGregorianCalendar> dateTimeFilter =
(PropertyValueFilter<XMLGregorianCalendar>) ((PropertyValueFilter<?>) filter);
(PropertyValueFilter<XMLGregorianCalendar>) filter;
return processString(extItem,
ValueFilterValues.from(dateTimeFilter, ExtUtils::extensionDateTime),
operation, filter);
} else if (extItem.valueType.equals(POLY_STRING_TYPE)) {
return processPolyString(extItem, values, operation, filter);
} else if (extItem.valueType.equals(REF_TYPE)) {
// TODO
return processPolyString(extItem, values, operation, propertyValueFilter);
}

throw new QueryException("Unsupported filter for extension item: " + filter);
}

private Predicate processReference(MExtItem extItem, RefFilter filter) {
List<PrismReferenceValue> values = filter.getValues();
if (values == null || values.isEmpty()) {
return extItemIsNull(extItem);
}

if (values.size() == 1) {
return processSingleReferenceValue(extItem, filter, values.get(0));
}

Predicate predicate = null;
for (PrismReferenceValue ref : values) {
predicate = ExpressionUtils.or(predicate,
processSingleReferenceValue(extItem, filter, ref));
}
return predicate;
}

private Predicate processSingleReferenceValue(MExtItem extItem, RefFilter filter, PrismReferenceValue ref) {
// We always store oid+type+relation, nothing is null or the whole item is null => missing.
// So if we ask for OID IS NULL or for target type IS NULL we actually ask "item IS NULL".
if (ref.getOid() == null && !filter.isOidNullAsAny()
|| ref.getTargetType() == null && !filter.isTargetTypeNullAsAny()) {
return extItemIsNull(extItem);
}

StringBuilder json = new StringBuilder("{");
boolean commaNeeded = false;

if (ref.getOid() != null) {
json.append("\"o\":\"").append(ref.getOid()).append("\"");
commaNeeded = true;
}

if (ref.getTargetType() != null) {
MObjectType objectType = MObjectType.fromTypeQName(ref.getTargetType());
if (commaNeeded) {
json.append(',');
}
json.append("\"t\":\"").append(objectType).append("\"");
commaNeeded = true;
}

if (ref.getRelation() == null || !ref.getRelation().equals(PrismConstants.Q_ANY)) {
Integer relationId = ((SqaleQueryContext<?, ?, ?>) context)
.searchCachedRelationId(ref.getRelation());
if (commaNeeded) {
json.append(',');
}
json.append("\"r\":").append(relationId);
} else {
// relation == Q_ANY, no additional predicate needed
}

// closing } for inner object is in String.format
return predicateWithNotTreated(path, booleanTemplate("{0} @> {1}::jsonb", path,
String.format("{\"%d\":%s}}", extItem.id, json)));
}

private Predicate processString(
MExtItem extItem, ValueFilterValues<?, ?> values, FilterOperation operation, ObjectFilter filter)
throws QueryException {
Expand Down Expand Up @@ -212,6 +275,7 @@ private Predicate processBoolean(
}

// filter should be PropertyValueFilter<PolyString>, but pure Strings are handled fine

private Predicate processPolyString(MExtItem extItem, ValueFilterValues<?, ?> values,
FilterOperation operation, PropertyValueFilter<?> filter)
throws QueryException {
Expand Down Expand Up @@ -293,4 +357,12 @@ protected boolean isIgnoreCaseFilter(ValueFilter<?, ?> filter) {
|| ORIG_IGNORE_CASE.equals(matchingRule)
|| NORM_IGNORE_CASE.equals(matchingRule);
}

private BooleanExpression extItemIsNull(MExtItem extItem) {
// ?? is "escaped" ? operator, PG JDBC driver understands it. Alternative is to use
// function jsonb_exists but that does NOT use GIN index, only operators do!
// We have to use parenthesis with AND shovelled into the template like this.
return booleanTemplate("({0} ?? {1} AND {0} is not null)",
path, extItem.id.toString()).not();
}
}
17 changes: 10 additions & 7 deletions ...qale/src/main/java/com/evolveum/midpoint/repo/sqale/filtering/RefItemFilterProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,15 @@ private Predicate processSingleValue(RefFilter filter, PrismReferenceValue ref)
} else if (!filter.isOidNullAsAny()) {
predicate = oidPath.isNull();
}

if (ref.getTargetType() != null) {
MObjectType objectType = MObjectType.fromTypeQName(ref.getTargetType());
predicate = ExpressionUtils.and(predicate,
predicateWithNotTreated(typePath, typePath.eq(objectType)));
} else if (!filter.isTargetTypeNullAsAny()) {
predicate = ExpressionUtils.and(predicate, typePath.isNull());
}

if (ref.getRelation() == null || !ref.getRelation().equals(PrismConstants.Q_ANY)) {
Integer relationId = ((SqaleQueryContext<?, ?, ?>) context)
.searchCachedRelationId(ref.getRelation());
Expand All @@ -92,13 +101,7 @@ private Predicate processSingleValue(RefFilter filter, PrismReferenceValue ref)
} else {
// relation == Q_ANY, no additional predicate needed
}
if (ref.getTargetType() != null) {
MObjectType objectType = MObjectType.fromTypeQName(ref.getTargetType());
predicate = ExpressionUtils.and(predicate,
predicateWithNotTreated(typePath, typePath.eq(objectType)));
} else if (!filter.isTargetTypeNullAsAny()) {
predicate = ExpressionUtils.and(predicate, typePath.isNull());
}

return predicate;
}
}
66 changes: 60 additions & 6 deletions ...-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSearchObjectTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,22 +1104,22 @@ public void test532BigDecimalRepresentationDoesNotMatterToDbOnlyValue() throws S

@Test
public void test533SearchObjectHavingSpecifiedDoubleExtension() throws SchemaException {
searchUsersTest("\"having extension double item equal to big decimal value\"",
searchUsersTest("having extension double item equal to big decimal value",
f -> f.item(UserType.F_EXTENSION, new QName("double")).eq(Double.MAX_VALUE),
user1Oid);
}

@Test
public void test534SearchObjectHavingNumericExtItemBetweenTwoValues() throws SchemaException {
searchUsersTest("\"having extension numeric item between two value\"",
searchUsersTest("having extension numeric item between two value",
f -> f.item(UserType.F_EXTENSION, new QName("int")).gt(0)
.and().item(UserType.F_EXTENSION, new QName("int")).lt(3),
user1Oid, user2Oid);
}

@Test
public void test535SearchObjectHavingNumericExtItemUsingGoeAndBigInteger() throws SchemaException {
searchUsersTest("\"having extension double item equal to big decimal value\"",
searchUsersTest("having extension double item equal to big decimal value",
f -> f.item(UserType.F_EXTENSION, new QName("double")).ge(
new BigInteger("17976931348623157000000000000000000000000000000000000000000"
+ "0000000000000000000000000000000000000000000000000000000000000000"
Expand All @@ -1131,20 +1131,43 @@ public void test535SearchObjectHavingNumericExtItemUsingGoeAndBigInteger() throw

@Test
public void test536SearchObjectHavingDoubleExtItemBetweenTwoValues() throws SchemaException {
searchUsersTest("\"having extension double item equal to big decimal value\"",
searchUsersTest("having extension double item between two values",
f -> f.item(UserType.F_EXTENSION, new QName("double")).gt(0)
.and().item(UserType.F_EXTENSION, new QName("double")).lt(3d),
user2Oid);
}

@Test
public void test537SearchObjectHavingFloatOrShortExtItem() throws SchemaException {
searchUsersTest("\"having extension double item equal to big decimal value\"",
searchUsersTest("having either float or short extension with specified conditions",
f -> f.item(UserType.F_EXTENSION, new QName("float")).gt(-1f)
.or().item(UserType.F_EXTENSION, new QName("short")).lt((short) 4),
user1Oid, user2Oid);
}

@Test
public void test538SearchObjectHavingShortGoe() throws SchemaException {
searchUsersTest("having extension short item greater than or equal to value",
f -> f.item(UserType.F_EXTENSION, new QName("short")).ge((short) 3),
user1Oid);
}

@Test
public void test539SearchObjectHavingShortGoe() throws SchemaException {
/*
This test assures that users with non-null ext column without any "short" item are found
too, even for operations not using containment operators (@> or ?).
If the condition inside the NOT is "(u.ext->'3')::numeric >= $1" than there are two ways
how to inverse the NOT properly:
- default, but naive approach: not ((u.ext->'3')::numeric >= $1 and (u.ext->'3')::numeric is not null)
- better, requiring a fix: not ((u.ext->'3')::numeric >= $1 and ext ? '3' and ext is not null)
Good thing is that by default it works using the first approach, although not ideal.
*/
searchUsersTest("having extension short item greater than or equal to value",
f -> f.not().item(UserType.F_EXTENSION, new QName("short")).ge((short) 3),
user2Oid, user3Oid, user4Oid);
}

// enum tests
@Test
public void test540SearchObjectHavingSpecifiedEnumExtension() throws SchemaException {
Expand Down Expand Up @@ -1350,8 +1373,39 @@ public void test571SearchObjectWithExtensionMultiValuePolyStringCaseIgnoreFails(
.hasMessageContaining("supported");
}

// TODO ref extension query
@Test
public void test580SearchObjectWithExtensionRef() throws SchemaException {
searchUsersTest("extension ref item matching",
f -> f.item(UserType.F_EXTENSION, new QName("ref"))
.ref(ref(org21Oid, OrgType.COMPLEX_TYPE, relation1)),
user1Oid);
}

@Test
public void test581SearchObjectWithExtensionRefByOidOnly() throws SchemaException {
searchUsersTest("extension ref item matching by OID only (implies default relation)",
f -> f.item(UserType.F_EXTENSION, new QName("ref"))
.ref(org21Oid));
// used ref has non-default relation, so, correctly, it is not found
}

@Test
public void test582SearchObjectWithExtensionRefByOidOnly() throws SchemaException {
searchUsersTest("extension ref item matching by OID only",
f -> f.item(UserType.F_EXTENSION, new QName("ref"))
.ref(ref(org21Oid, null, PrismConstants.Q_ANY)),
user1Oid);
}

@Test
public void test583SearchObjectWithExtensionRefByUnusedOid() throws SchemaException {
searchUsersTest("extension ref item matching by unused OID",
f -> f.item(UserType.F_EXTENSION, new QName("ref"))
.ref(org12Oid));
}

// TODO multi-value EQ filter (IN semantics) is not supported YET
// TODO shadow attribute test, just few, otherwise it's like extension
// endregion

// region special cases
Expand Down
6 changes: 0 additions & 6 deletions .../src/main/java/com/evolveum/midpoint/repo/sqlbase/filtering/item/ItemFilterProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,6 @@ protected <T> Predicate createBinaryCondition(
* otherwise the expression is passed as-is.
* Technically, any expression can be used on path side as well.
*/
protected Predicate singleValuePredicate(Expression<?> path, Ops operator, Object value) {
Predicate predicate = ExpressionUtils.predicate(operator, path,
value instanceof Expression ? (Expression<?>) value : ConstantImpl.create(value));
return predicateWithNotTreated(path, predicate);
}

protected Predicate singleValuePredicate(
Expression<?> path, FilterOperation operation, Object value) {
path = operation.treatPath(path);
Expand Down

0 comments on commit 5e1d82b

Please sign in to comment.