Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ public boolean test(T value) {
return String.valueOf(value).startsWith((String) literal.value());
case NOT_STARTS_WITH:
return !String.valueOf(value).startsWith((String) literal.value());
case ENDS_WITH:
return String.valueOf(value).endsWith((String) literal.value());
case NOT_ENDS_WITH:
return !String.valueOf(value).endsWith((String) literal.value());
default:
throw new IllegalStateException("Invalid operation for BoundLiteralPredicate: " + op());
}
Expand Down Expand Up @@ -159,6 +163,10 @@ public String toString() {
return term() + " startsWith \"" + literal + "\"";
case NOT_STARTS_WITH:
return term() + " notStartsWith \"" + literal + "\"";
case ENDS_WITH:
return term() + " endsWith \"" + literal + "\"";
case NOT_ENDS_WITH:
return term() + " notEndsWith \"" + literal + "\"";
case IN:
return term() + " in { " + literal + " }";
case NOT_IN:
Expand Down
11 changes: 11 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Evaluator.java
Original file line number Diff line number Diff line change
Expand Up @@ -156,5 +156,16 @@ public <T> Boolean startsWith(Bound<T> valueExpr, Literal<T> lit) {
public <T> Boolean notStartsWith(Bound<T> valueExpr, Literal<T> lit) {
return !startsWith(valueExpr, lit);
}

@Override
public <T> Boolean endsWith(Bound<T> valueExpr, Literal<T> lit) {
T evalRes = valueExpr.eval(struct);
return evalRes != null && ((String) evalRes).endsWith((String) lit.value());
}

@Override
public <T> Boolean notEndsWith(Bound<T> valueExpr, Literal<T> lit) {
return !endsWith(valueExpr, lit);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ enum Operation {
OR,
STARTS_WITH,
NOT_STARTS_WITH,
ENDS_WITH,
NOT_ENDS_WITH,
COUNT,
COUNT_NULL,
COUNT_STAR,
Expand Down Expand Up @@ -91,6 +93,10 @@ public Operation negate() {
return Operation.NOT_STARTS_WITH;
case NOT_STARTS_WITH:
return Operation.STARTS_WITH;
case ENDS_WITH:
return Operation.NOT_ENDS_WITH;
case NOT_ENDS_WITH:
return Operation.ENDS_WITH;
default:
throw new IllegalArgumentException("No negation for operation: " + this);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,8 @@ public <T> Expression predicate(UnboundPredicate<T> pred) {
case NOT_EQ:
case STARTS_WITH:
case NOT_STARTS_WITH:
case ENDS_WITH:
case NOT_ENDS_WITH:
return new UnboundPredicate<>(
pred.op(), pred.term(), (T) sanitize(pred.literal(), now, today));
case IN:
Expand Down Expand Up @@ -441,6 +443,10 @@ public <T> String predicate(BoundPredicate<T> pred) {
return term + " STARTS WITH " + value((BoundLiteralPredicate<?>) pred);
case NOT_STARTS_WITH:
return term + " NOT STARTS WITH " + value((BoundLiteralPredicate<?>) pred);
case ENDS_WITH:
return term + " ENDS WITH " + value((BoundLiteralPredicate<?>) pred);
case NOT_ENDS_WITH:
return term + " NOT ENDS WITH " + value((BoundLiteralPredicate<?>) pred);
default:
throw new UnsupportedOperationException(
"Cannot sanitize unsupported predicate type: " + pred.op());
Expand Down Expand Up @@ -493,6 +499,10 @@ public <T> String predicate(UnboundPredicate<T> pred) {
return term + " STARTS WITH " + sanitize(pred.literal(), nowMicros, today);
case NOT_STARTS_WITH:
return term + " NOT STARTS WITH " + sanitize(pred.literal(), nowMicros, today);
case ENDS_WITH:
return term + " ENDS WITH " + sanitize(pred.literal(), nowMicros, today);
case NOT_ENDS_WITH:
return term + " NOT ENDS WITH " + sanitize(pred.literal(), nowMicros, today);
default:
throw new UnsupportedOperationException(
"Cannot sanitize unsupported predicate type: " + pred.op());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,16 @@ public <T> R notStartsWith(BoundReference<T> ref, Literal<T> lit) {
"notStartsWith expression is not supported by the visitor");
}

public <T> R endsWith(BoundReference<T> ref, Literal<T> lit) {
throw new UnsupportedOperationException(
"endsWith expression is not supported by the visitor");
}

public <T> R notEndsWith(BoundReference<T> ref, Literal<T> lit) {
throw new UnsupportedOperationException(
"notEndsWith expression is not supported by the visitor");
}

/**
* Handle a non-reference value in this visitor.
*
Expand Down Expand Up @@ -166,6 +176,10 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith((BoundReference<T>) pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith((BoundReference<T>) pred.term(), literalPred.literal());
case ENDS_WITH:
return endsWith((BoundReference<T>) pred.term(), literalPred.literal());
case NOT_ENDS_WITH:
return notEndsWith((BoundReference<T>) pred.term(), literalPred.literal());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -266,6 +280,14 @@ public <T> R notStartsWith(Bound<T> expr, Literal<T> lit) {
throw new UnsupportedOperationException("Unsupported operation.");
}

public <T> R endsWith(Bound<T> expr, Literal<T> lit) {
throw new UnsupportedOperationException("Unsupported operation.");
}

public <T> R notEndsWith(Bound<T> expr, Literal<T> lit) {
throw new UnsupportedOperationException("Unsupported operation.");
}

@Override
public <T> R predicate(BoundPredicate<T> pred) {
if (pred.isLiteralPredicate()) {
Expand All @@ -287,6 +309,10 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith(pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith(pred.term(), literalPred.literal());
case ENDS_WITH:
return endsWith(pred.term(), literalPred.literal());
case NOT_ENDS_WITH:
return notEndsWith(pred.term(), literalPred.literal());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -465,6 +491,10 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith(pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith(pred.term(), literalPred.literal());
case ENDS_WITH:
return endsWith(pred.term(), literalPred.literal());
case NOT_ENDS_WITH:
return notEndsWith(pred.term(), literalPred.literal());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -555,6 +585,14 @@ public <T> R startsWith(BoundTerm<T> term, Literal<T> lit) {
public <T> R notStartsWith(BoundTerm<T> term, Literal<T> lit) {
return null;
}

public <T> R endsWith(BoundTerm<T> term, Literal<T> lit) {
return null;
}

public <T> R notEndsWith(BoundTerm<T> term, Literal<T> lit) {
return null;
}
}

/**
Expand Down
16 changes: 16 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Expressions.java
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,22 @@ public static UnboundPredicate<String> notStartsWith(UnboundTerm<String> expr, S
return new UnboundPredicate<>(Expression.Operation.NOT_STARTS_WITH, expr, value);
}

public static UnboundPredicate<String> endsWith(String name, String value) {
return new UnboundPredicate<>(Expression.Operation.ENDS_WITH, ref(name), value);
}

public static UnboundPredicate<String> endsWith(UnboundTerm<String> expr, String value) {
return new UnboundPredicate<>(Expression.Operation.ENDS_WITH, expr, value);
}

public static UnboundPredicate<String> notEndsWith(String name, String value) {
return new UnboundPredicate<>(Expression.Operation.NOT_ENDS_WITH, ref(name), value);
}

public static UnboundPredicate<String> notEndsWith(UnboundTerm<String> expr, String value) {
return new UnboundPredicate<>(Expression.Operation.NOT_ENDS_WITH, expr, value);
}

public static <T> UnboundPredicate<T> in(String name, T... values) {
return predicate(Operation.IN, name, Lists.newArrayList(values));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,22 @@ public <T> Boolean notStartsWith(Bound<T> term, Literal<T> lit) {
return ROWS_MIGHT_MATCH;
}

@Override
public <T> Boolean endsWith(Bound<T> term, Literal<T> lit) {
// the only transforms that produce strings are truncate and identity, which work with this
int id = term.ref().fieldId();
if (containsNullsOnly(id)) {
return ROWS_CANNOT_MATCH;
}

return ROWS_MIGHT_MATCH;
}

@Override
public <T> Boolean notEndsWith(Bound<T> term, Literal<T> lit) {
return ROWS_MIGHT_MATCH;
}

private boolean mayContainNull(Integer id) {
return nullCounts == null || !nullCounts.containsKey(id) || nullCounts.get(id) != 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,22 @@ public <T> Boolean notStartsWith(BoundReference<T> ref, Literal<T> lit) {
return ROWS_MIGHT_MATCH;
}

@Override
public <T> Boolean endsWith(BoundReference<T> ref, Literal<T> lit) {
int pos = Accessors.toPosition(ref.accessor());
PartitionFieldSummary fieldStats = stats.get(pos);
if (fieldStats.lowerBound() == null) {
return ROWS_CANNOT_MATCH; // values are all null, cannot end with the literal
}

return ROWS_MIGHT_MATCH;
}

@Override
public <T> Boolean notEndsWith(BoundReference<T> ref, Literal<T> lit) {
return ROWS_MIGHT_MATCH;
}

private boolean allValuesAreNull(PartitionFieldSummary summary, Type.TypeID typeId) {
// containsNull encodes whether at least one partition value is null,
// lowerBound is null if all partition values are null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,20 @@ public <T> Expression notStartsWith(BoundReference<T> ref, Literal<T> lit) {
: alwaysTrue();
}

@Override
public <T> Expression endsWith(BoundReference<T> ref, Literal<T> lit) {
return ((String) ref.eval(struct)).endsWith((String) lit.value())
? alwaysTrue()
: alwaysFalse();
}

@Override
public <T> Expression notEndsWith(BoundReference<T> ref, Literal<T> lit) {
return ((String) ref.eval(struct)).endsWith((String) lit.value())
? alwaysFalse()
: alwaysTrue();
}

@Override
@SuppressWarnings("unchecked")
public <T> Expression predicate(BoundPredicate<T> pred) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,16 @@ public <T> Boolean notStartsWith(BoundReference<T> ref, Literal<T> lit) {
return ROWS_MIGHT_NOT_MATCH;
}

@Override
public <T> Boolean endsWith(BoundReference<T> ref, Literal<T> lit) {
return ROWS_MIGHT_NOT_MATCH;
}

@Override
public <T> Boolean notEndsWith(BoundReference<T> ref, Literal<T> lit) {
return ROWS_MIGHT_NOT_MATCH;
}

private boolean isNestedColumn(int id) {
return struct.field(id) == null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@ private boolean floatingType(Type.TypeID typeID) {
}

private Expression bindLiteralOperation(BoundTerm<T> boundTerm) {
if (op() == Operation.STARTS_WITH || op() == Operation.NOT_STARTS_WITH) {
if (op() == Operation.STARTS_WITH
|| op() == Operation.NOT_STARTS_WITH
|| op() == Operation.ENDS_WITH
|| op() == Operation.NOT_ENDS_WITH) {
ValidationException.check(
boundTerm.type().equals(Types.StringType.get()),
"Term for STARTS_WITH or NOT_STARTS_WITH must produce a string: %s: %s",
"Term for %s must produce a string: %s: %s",
op().name(),
boundTerm,
boundTerm.type());
}
Expand Down Expand Up @@ -284,6 +288,10 @@ public String toString() {
return term() + " startsWith \"" + literal() + "\"";
case NOT_STARTS_WITH:
return term() + " notStartsWith \"" + literal() + "\"";
case ENDS_WITH:
return term() + " endsWith \"" + literal() + "\"";
case NOT_ENDS_WITH:
return term() + " notEndsWith \"" + literal() + "\"";
case IN:
return term() + " in (" + COMMA.join(literals()) + ")";
case NOT_IN:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.apache.iceberg.expressions.Expressions.alwaysFalse;
import static org.apache.iceberg.expressions.Expressions.alwaysTrue;
import static org.apache.iceberg.expressions.Expressions.and;
import static org.apache.iceberg.expressions.Expressions.endsWith;
import static org.apache.iceberg.expressions.Expressions.equal;
import static org.apache.iceberg.expressions.Expressions.greaterThan;
import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
Expand All @@ -30,6 +31,7 @@
import static org.apache.iceberg.expressions.Expressions.lessThan;
import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual;
import static org.apache.iceberg.expressions.Expressions.not;
import static org.apache.iceberg.expressions.Expressions.notEndsWith;
import static org.apache.iceberg.expressions.Expressions.notEqual;
import static org.apache.iceberg.expressions.Expressions.notIn;
import static org.apache.iceberg.expressions.Expressions.notNaN;
Expand Down Expand Up @@ -348,6 +350,54 @@ public void testNotStartsWith() {
.isTrue();
}

@Test
public void testEndsWith() {
StructType struct = StructType.of(required(24, "s", Types.StringType.get()));
Evaluator evaluator = new Evaluator(struct, endsWith("s", "abc"));
assertThat(evaluator.eval(TestHelpers.Row.of("abc")))
.as("abc endsWith abc should be true")
.isTrue();
assertThat(evaluator.eval(TestHelpers.Row.of("abcx")))
.as("abcx endsWith abc should be false")
.isFalse();
assertThat(evaluator.eval(TestHelpers.Row.of("Abc")))
.as("Abc endsWith abc should be false")
.isFalse();
assertThat(evaluator.eval(TestHelpers.Row.of("c")))
.as("c endsWith abc should be false")
.isFalse();
assertThat(evaluator.eval(TestHelpers.Row.of("xyzabc")))
.as("xyzabc endsWith abc should be true")
.isTrue();
assertThat(evaluator.eval(TestHelpers.Row.of((String) null)))
.as("null endsWith abc should be false")
.isFalse();
}

@Test
public void testNotEndsWith() {
StructType struct = StructType.of(required(24, "s", Types.StringType.get()));
Evaluator evaluator = new Evaluator(struct, notEndsWith("s", "abc"));
assertThat(evaluator.eval(TestHelpers.Row.of("abc")))
.as("abc notEndsWith abc should be false")
.isFalse();
assertThat(evaluator.eval(TestHelpers.Row.of("abcx")))
.as("abcx notEndsWith abc should be true")
.isTrue();
assertThat(evaluator.eval(TestHelpers.Row.of("Abc")))
.as("Abc notEndsWith abc should be true")
.isTrue();
assertThat(evaluator.eval(TestHelpers.Row.of("c")))
.as("c notEndsWith abc should be true")
.isTrue();
assertThat(evaluator.eval(TestHelpers.Row.of("xyzabc")))
.as("xyzabc notEndsWith abc should be false")
.isFalse();
assertThat(evaluator.eval(TestHelpers.Row.of("XyzAbc")))
.as("XyzAbc notEndsWith abc should be true")
.isTrue();
}

@Test
public void testAlwaysTrue() {
Evaluator evaluator = new Evaluator(STRUCT, alwaysTrue());
Expand Down
Loading