From d6d8d6d775bd1a9101d31ba28ac8babea74b1868 Mon Sep 17 00:00:00 2001 From: Jeff Schnitzer Date: Fri, 23 Sep 2022 16:57:17 -0700 Subject: [PATCH] Implement complex filters, including nested OR/AND composites --- .../com/googlecode/objectify/cmd/Filter.java | 267 ++++++++++++++++++ .../com/googlecode/objectify/cmd/Query.java | 14 +- .../objectify/impl/FilterOperator.java | 2 +- .../objectify/impl/LoadTypeImpl.java | 10 +- .../objectify/impl/ObjectifyImpl.java | 2 +- .../googlecode/objectify/impl/QueryImpl.java | 20 +- .../objectify/test/QueryBasicTests.java | 13 + .../test/QueryComplexFilterTests.java | 110 ++++++++ .../objectify/test/RemoteQueryTests.java | 28 ++ 9 files changed, 460 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/googlecode/objectify/cmd/Filter.java create mode 100644 src/test/java/com/googlecode/objectify/test/QueryComplexFilterTests.java diff --git a/src/main/java/com/googlecode/objectify/cmd/Filter.java b/src/main/java/com/googlecode/objectify/cmd/Filter.java new file mode 100644 index 000000000..6debc9d3c --- /dev/null +++ b/src/main/java/com/googlecode/objectify/cmd/Filter.java @@ -0,0 +1,267 @@ +package com.googlecode.objectify.cmd; + +import com.google.cloud.datastore.StructuredQuery; +import com.google.cloud.datastore.Value; +import com.google.common.base.Preconditions; +import com.googlecode.objectify.impl.FilterOperator; +import com.googlecode.objectify.impl.ObjectifyImpl; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.experimental.NonFinal; + +/** + * Gives us the ability to compose arbitrarily complex filters with OR and AND sections. + */ +abstract public class Filter { + /** + * Create a filter condition that requires the property to equal the scalar value. + */ + public static Filter equalTo(final String property, final Object value) { + return new EqualToFilter(property, value); + } + + /** + * Create a filter condition that requires the property to not be equal to the scalar value. + */ + public static Filter notEqualTo(final String property, final Object value) { + return new NotEqualToFilter(property, value); + } + + /** + * Create a filter condition that requires the property to be greater than the scalar value + */ + public static Filter greaterThan(final String property, final Object value) { + return new GreaterThanFilter(property, value); + } + + /** + * Create a filter condition that requires the property to be greater than or equal to scalar value + */ + public static Filter greaterThanOrEqualTo(final String property, final Object value) { + return new GreaterThanOrEqualToFilter(property, value); + } + + /** + * Create a filter condition that requires the property to be less than the scalar value + */ + public static Filter lessThan(final String property, final Object value) { + return new LessThanFilter(property, value); + } + + /** + * Create a filter condition that requires the property to be less than or equal to the scalar value + */ + public static Filter lessThanOrEqualTo(final String property, final Object value) { + return new LessThanOrEqualToFilter(property, value); + } + + /** + * Create a filter condition that requires the property to be equal to at least one of a list of scalar + * values. + * + * @param value must be an array or collection + */ + public static Filter in(final String property, final Object value) { + return new InFilter(property, value); + } + + /** + * Create a filter condition that requires the property to NOT be equal to any of a list of scalar + * values. + * + * @param value must be an array or collection + */ + public static Filter notIn(final String property, final Object value) { + return new NotInFilter(property, value); + } + + /** + * Combine an arbitrary list of filter conditions. They can be nested. + */ + public static Filter or(final Filter... filters) { + return new OrFilter(filters); + } + + /** + * Combine an arbitrary list of filter conditions. They can be nested. + */ + public static Filter and(final Filter... filters) { + return new AndFilter(filters); + } + + /** + * Convert to a low level API filter. + */ + abstract public StructuredQuery.Filter convert(final ObjectifyImpl ofyImpl); + + @lombok.Value @NonFinal + @EqualsAndHashCode(callSuper = false) + abstract private static class PropertyFilter extends Filter { + String property; + Object value; + + @Override + public final StructuredQuery.Filter convert(final ObjectifyImpl ofyImpl) { + final Value raw = ofyImpl.makeFilterable(getValue()); + return convert(raw); + } + + abstract protected StructuredQuery.Filter convert(final Value rawValue); + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class EqualToFilter extends PropertyFilter { + public EqualToFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.EQUAL.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class NotEqualToFilter extends PropertyFilter { + public NotEqualToFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.NOT_EQUAL.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class GreaterThanFilter extends PropertyFilter { + public GreaterThanFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.GREATER_THAN.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class GreaterThanOrEqualToFilter extends PropertyFilter { + public GreaterThanOrEqualToFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.GREATER_THAN_OR_EQUAL.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class LessThanFilter extends PropertyFilter { + public LessThanFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.LESS_THAN.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class LessThanOrEqualToFilter extends PropertyFilter { + public LessThanOrEqualToFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.LESS_THAN_OR_EQUAL.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class InFilter extends PropertyFilter { + public InFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.IN.of(getProperty(), rawValue); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class NotInFilter extends PropertyFilter { + public NotInFilter(final String property, final Object value) { + super(property, value); + } + + @Override + protected StructuredQuery.Filter convert(final Value rawValue) { + return FilterOperator.NOT_IN.of(getProperty(), rawValue); + } + } + + @lombok.Value @NonFinal + @EqualsAndHashCode(callSuper = false) + abstract private static class CompositeFilter extends Filter { + Filter[] filters; + + public CompositeFilter(final Filter[] filters) { + Preconditions.checkArgument(filters.length >= 1, "You must include at least one condition"); + this.filters = filters; + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class AndFilter extends CompositeFilter { + public AndFilter(final Filter[] filters) { + super(filters); + } + + @Override + public StructuredQuery.Filter convert(final ObjectifyImpl ofyImpl) { + final StructuredQuery.Filter first = getFilters()[0].convert(ofyImpl); + + final StructuredQuery.Filter[] rest = new StructuredQuery.Filter[getFilters().length - 1]; + for (int i = 1; i < getFilters().length; i++) { + rest[i - 1] = getFilters()[i].convert(ofyImpl); + } + + return StructuredQuery.CompositeFilter.and(first, rest); + } + } + + @EqualsAndHashCode(callSuper = true) + @ToString(callSuper = true) + private static class OrFilter extends CompositeFilter { + public OrFilter(final Filter[] filters) { + super(filters); + } + + @Override + public StructuredQuery.Filter convert(final ObjectifyImpl ofyImpl) { + final StructuredQuery.Filter first = getFilters()[0].convert(ofyImpl); + + final StructuredQuery.Filter[] rest = new StructuredQuery.Filter[getFilters().length - 1]; + for (int i = 1; i < getFilters().length; i++) { + rest[i - 1] = getFilters()[i].convert(ofyImpl); + } + + //return StructuredQuery.CompositeFilter.or(first, rest); + throw new UnsupportedOperationException("OR is not yet available in the low-level API. Coming soon."); + } + } +} diff --git a/src/main/java/com/googlecode/objectify/cmd/Query.java b/src/main/java/com/googlecode/objectify/cmd/Query.java index f34e80922..95d22cdf3 100644 --- a/src/main/java/com/googlecode/objectify/cmd/Query.java +++ b/src/main/java/com/googlecode/objectify/cmd/Query.java @@ -1,7 +1,7 @@ package com.googlecode.objectify.cmd; import com.google.cloud.datastore.Cursor; -import com.google.cloud.datastore.StructuredQuery.Filter; +import com.google.cloud.datastore.StructuredQuery; /** @@ -55,6 +55,18 @@ public interface Query extends SimpleQuery *

You can not filter on @Id or @Parent properties. Use * {@code filterKey()} or {@code ancestor()} instead.

*/ + public Query filter(StructuredQuery.Filter filter); + + /** + *

Create an arbitrarily complex filter. This method is preferred to the low-level Filter method + * because it has better ergonomics and automatically handles objects like Objectify {@code Key} + * and {@code Ref}.

+ * + *

Construct Filter objects using static methods on the Filter class.

+ * + *

Note that like the other filter methods, you can not filter on @Id or @Parent properties. + * You can filter by {@code __key__} however.

+ */ public Query filter(Filter filter); /* (non-Javadoc) diff --git a/src/main/java/com/googlecode/objectify/impl/FilterOperator.java b/src/main/java/com/googlecode/objectify/impl/FilterOperator.java index 616ea6cf7..bc8e4c40c 100644 --- a/src/main/java/com/googlecode/objectify/impl/FilterOperator.java +++ b/src/main/java/com/googlecode/objectify/impl/FilterOperator.java @@ -9,7 +9,7 @@ /** The cloud sdk filtering API is somewhat hostile to programmatic query generation, so we need this adaptor */ @RequiredArgsConstructor -enum FilterOperator { +public enum FilterOperator { LESS_THAN(PropertyFilter::lt), LESS_THAN_OR_EQUAL(PropertyFilter::le), GREATER_THAN(PropertyFilter::gt), diff --git a/src/main/java/com/googlecode/objectify/impl/LoadTypeImpl.java b/src/main/java/com/googlecode/objectify/impl/LoadTypeImpl.java index 57b842f41..bf22d6641 100644 --- a/src/main/java/com/googlecode/objectify/impl/LoadTypeImpl.java +++ b/src/main/java/com/googlecode/objectify/impl/LoadTypeImpl.java @@ -1,9 +1,10 @@ package com.googlecode.objectify.impl; -import com.google.cloud.datastore.StructuredQuery.Filter; +import com.google.cloud.datastore.StructuredQuery; import com.googlecode.objectify.Key; import com.googlecode.objectify.LoadResult; import com.googlecode.objectify.ObjectifyFactory; +import com.googlecode.objectify.cmd.Filter; import com.googlecode.objectify.cmd.LoadIds; import com.googlecode.objectify.cmd.LoadType; import com.googlecode.objectify.cmd.Query; @@ -64,6 +65,13 @@ public Query filter(String condition, Object value) { } /* */ + @Override + public Query filter(final StructuredQuery.Filter filter) { + final QueryImpl q = createQuery(); + q.addFilter(filter); + return q; + } + @Override public Query filter(final Filter filter) { final QueryImpl q = createQuery(); diff --git a/src/main/java/com/googlecode/objectify/impl/ObjectifyImpl.java b/src/main/java/com/googlecode/objectify/impl/ObjectifyImpl.java index 51cb1edd3..2392db12a 100644 --- a/src/main/java/com/googlecode/objectify/impl/ObjectifyImpl.java +++ b/src/main/java/com/googlecode/objectify/impl/ObjectifyImpl.java @@ -266,7 +266,7 @@ protected WriteEngine createWriteEngine() { * * @return whatever can be put into a filter clause. */ - protected Value makeFilterable(Object value) { + public Value makeFilterable(Object value) { if (value == null) return NullValue.of(); diff --git a/src/main/java/com/googlecode/objectify/impl/QueryImpl.java b/src/main/java/com/googlecode/objectify/impl/QueryImpl.java index 403640baa..c2795847b 100644 --- a/src/main/java/com/googlecode/objectify/impl/QueryImpl.java +++ b/src/main/java/com/googlecode/objectify/impl/QueryImpl.java @@ -4,7 +4,7 @@ import com.google.cloud.datastore.KeyQuery; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.StringValue; -import com.google.cloud.datastore.StructuredQuery.Filter; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.Value; @@ -13,6 +13,7 @@ import com.googlecode.objectify.LoadResult; import com.googlecode.objectify.ObjectifyFactory; import com.googlecode.objectify.annotation.Subclass; +import com.googlecode.objectify.cmd.Filter; import com.googlecode.objectify.cmd.Query; import com.googlecode.objectify.cmd.QueryResultIterable; import com.googlecode.objectify.impl.translate.ClassTranslator; @@ -94,6 +95,13 @@ public QueryImpl filter(final String condition, final Object value) { } /* */ + @Override + public QueryImpl filter(final StructuredQuery.Filter filter) { + final QueryImpl q = createQuery(); + q.addFilter(filter); + return q; + } + @Override public QueryImpl filter(final Filter filter) { final QueryImpl q = createQuery(); @@ -144,10 +152,18 @@ else if (prop.equals(meta.getIdFieldName())) { /** * Add the filter as an AND to whatever is currently set as the actual filter. */ - void addFilter(final Filter filter) { + void addFilter(final StructuredQuery.Filter filter) { actual = actual.andFilter(filter); } + /** + * Add the filter as an AND to whatever is currently set as the actual filter. + */ + void addFilter(final Filter filter) { + final StructuredQuery.Filter munged = filter.convert(this.loader.getObjectifyImpl()); + actual = actual.andFilter(munged); + } + /** * Converts the textual operator (">", "<=", etc) into a FilterOperator. * Forgiving about the syntax; != and <> are NOT_EQUAL, = and == are EQUAL. diff --git a/src/test/java/com/googlecode/objectify/test/QueryBasicTests.java b/src/test/java/com/googlecode/objectify/test/QueryBasicTests.java index 75d2dc37a..c6e40911b 100644 --- a/src/test/java/com/googlecode/objectify/test/QueryBasicTests.java +++ b/src/test/java/com/googlecode/objectify/test/QueryBasicTests.java @@ -104,4 +104,17 @@ void queryByKindWithFilterWorks() throws Exception { final List fetched = ofy().load().kind(Key.getKind(Trivial.class)).filter("someString", "foo1").list(); assertThat(fetched).containsExactly(triv1); } + + /** */ + @Test + void simpleGreaterThanOnStringsWorks() throws Exception { + factory().register(Trivial.class); + + final Trivial triv1 = new Trivial(123L, "foo1", 12); + ofy().save().entities(triv1).now(); + ofy().clear(); + + final List fetched = ofy().load().type(Trivial.class).filter("someString >", "foo0").list(); + assertThat(fetched).containsExactly(triv1); + } } \ No newline at end of file diff --git a/src/test/java/com/googlecode/objectify/test/QueryComplexFilterTests.java b/src/test/java/com/googlecode/objectify/test/QueryComplexFilterTests.java new file mode 100644 index 000000000..5b1d05742 --- /dev/null +++ b/src/test/java/com/googlecode/objectify/test/QueryComplexFilterTests.java @@ -0,0 +1,110 @@ +/* + */ + +package com.googlecode.objectify.test; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.test.entity.Trivial; +import com.googlecode.objectify.test.util.TestBase; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.google.common.truth.Truth.assertThat; +import static com.googlecode.objectify.ObjectifyService.factory; +import static com.googlecode.objectify.ObjectifyService.ofy; +import static com.googlecode.objectify.cmd.Filter.and; +import static com.googlecode.objectify.cmd.Filter.equalTo; +import static com.googlecode.objectify.cmd.Filter.greaterThan; +import static com.googlecode.objectify.cmd.Filter.greaterThanOrEqualTo; +import static com.googlecode.objectify.cmd.Filter.lessThan; +import static com.googlecode.objectify.cmd.Filter.lessThanOrEqualTo; +import static com.googlecode.objectify.cmd.Filter.or; + +/** + * Exercise the Filter class. + */ +class QueryComplexFilterTests extends TestBase { + + /** */ + private Trivial triv1; + private Trivial triv2; + private Trivial triv3; + private List> keys; + + /** */ + @BeforeEach + void setUpExtra() { + factory().register(Trivial.class); + + this.triv1 = new Trivial("foo1", 1); + this.triv2 = new Trivial("foo2", 2); + this.triv3 = new Trivial("foo3", 3); + + final Map, Trivial> result = ofy().save().entities(triv1, triv2, triv3).now(); + + this.keys = new ArrayList<>(result.keySet()); + } + + /** */ + @Test + void simpleConditions() throws Exception { + { + final List list = ofy().load().type(Trivial.class).filter(equalTo("someString", "foo2")).list(); + assertThat(list).containsExactly(triv2); + } + + { + final List list = ofy().load().type(Trivial.class).filter(greaterThan("someString", "foo2")).list(); + assertThat(list).containsExactly(triv3); + } + + { + final List list = ofy().load().type(Trivial.class).filter(greaterThanOrEqualTo("someString", "foo2")).list(); + assertThat(list).containsExactly(triv2, triv3); + } + + { + final List list = ofy().load().type(Trivial.class).filter(lessThan("someString", "foo2")).list(); + assertThat(list).containsExactly(triv1); + } + + { + final List list = ofy().load().type(Trivial.class).filter(lessThanOrEqualTo("someString", "foo2")).list(); + assertThat(list).containsExactly(triv1, triv2); + } + } + + /** */ + @Test + void compositeAndConditions() throws Exception { + { + final List list = ofy().load().type(Trivial.class).filter( + and( + greaterThan("someString", "foo1"), + lessThan("someString", "foo3") + ) + ).list(); + assertThat(list).containsExactly(triv2); + } + } + + /** This doesn't work yet, need support in the low level API */ + @Test + @Disabled + void compositeOrConditions() throws Exception { + { + final List list = ofy().load().type(Trivial.class).filter( + or( + greaterThan("someString", "foo2"), + lessThan("someString", "foo2") + ) + ).list(); + assertThat(list).containsExactly(triv1, triv3); + } + } +} diff --git a/src/test/java/com/googlecode/objectify/test/RemoteQueryTests.java b/src/test/java/com/googlecode/objectify/test/RemoteQueryTests.java index 363d921da..f79b5ee11 100644 --- a/src/test/java/com/googlecode/objectify/test/RemoteQueryTests.java +++ b/src/test/java/com/googlecode/objectify/test/RemoteQueryTests.java @@ -24,6 +24,14 @@ import static com.google.common.truth.Truth.assertThat; import static com.googlecode.objectify.ObjectifyService.factory; import static com.googlecode.objectify.ObjectifyService.ofy; +import static com.googlecode.objectify.cmd.Filter.equalTo; +import static com.googlecode.objectify.cmd.Filter.greaterThan; +import static com.googlecode.objectify.cmd.Filter.greaterThanOrEqualTo; +import static com.googlecode.objectify.cmd.Filter.in; +import static com.googlecode.objectify.cmd.Filter.lessThan; +import static com.googlecode.objectify.cmd.Filter.lessThanOrEqualTo; +import static com.googlecode.objectify.cmd.Filter.notEqualTo; +import static com.googlecode.objectify.cmd.Filter.notIn; /** * The datastore emulator does not yet support certain operations. These run against a production database. @@ -116,4 +124,24 @@ void testNotIN() throws Exception { final List result = ofy().load().type(Trivial.class).filter("someString !in", conditions).list(); assertThat(result).containsExactly(triv1, triv2); } + + /** */ + @Test + void testFilters() throws Exception { + { + final List list = ofy().load().type(Trivial.class).filter(notEqualTo("someString", "foo2")).list(); + assertThat(list).containsExactly(triv1); + } + + { + final List list = ofy().load().type(Trivial.class).filter(in("someString", Arrays.asList("foo2", "foo3"))).list(); + assertThat(list).containsExactly(triv2); + } + + { + final List list = ofy().load().type(Trivial.class).filter(notIn("someString", Arrays.asList("foo2", "foo3"))).list(); + assertThat(list).containsExactly(triv1); + } + } + }