Skip to content

API, Core: Add geospatial predicates and bounding box literals #14101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kontinuation wants to merge 16 commits into
base: main
Choose a base branch
from
Open

API, Core: Add geospatial predicates and bounding box literals #14101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0dcf718
Add spatial expressions
Kontinuation Sep 18, 2025
9b4fdcf
Fix up after rebasing
Kontinuation Nov 18, 2025
d10d9d3
Fix some minor problems
Kontinuation Nov 18, 2025
9ec3e0a
Address review comments
Kontinuation Dec 11, 2025
622a101
Revert changes made to open-api SPEC, they were submitted as https://…
Kontinuation Dec 16, 2025
7f0c13f
Avoid mutating the value of byte buffer literal
Kontinuation Dec 16, 2025
6a0570b
Making BoundingBoxLiteral directly inherit from Literal<ByteBuffer>
Kontinuation Dec 16, 2025
8ac6daf
Addressed a few review comments
Kontinuation Dec 18, 2025
cb412b0
Revert expression parser changes and defer them to https://github.com…
Kontinuation Dec 18, 2025
1b7dc93
Fix more review comments
Kontinuation Dec 18, 2025
5bfa1dc
re-trigger tests
Kontinuation Dec 23, 2025
c5ada66
Normalize the byte buffer when constructing BoundingBoxLiteral
Kontinuation Jan 20, 2026
4ee66e7
Add more tests
Kontinuation Feb 3, 2026
3785d53
Change bounding box literal to extend from Literal<BoundingBox> inste…
Kontinuation Mar 14, 2026
0196496
Bring BoundingBoxLiteral comparator back to simplify code
Kontinuation Mar 14, 2026
4532bce
Further simplify the code
Kontinuation Mar 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Expression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ enum Operation {
OR,
STARTS_WITH,
NOT_STARTS_WITH,
ST_INTERSECTS,
ST_DISJOINT,
COUNT,
COUNT_NULL,
COUNT_STAR,
Expand Down Expand Up @@ -91,6 +93,10 @@ public Operation negate() {
return Operation.NOT_STARTS_WITH;
case NOT_STARTS_WITH:
return Operation.STARTS_WITH;
case ST_INTERSECTS:
return Operation.ST_DISJOINT;
case ST_DISJOINT:
return Operation.ST_INTERSECTS;
default:
throw new IllegalArgumentException("No negation for operation: " + this);
}
Expand Down
28 changes: 28 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.expressions.Literals.BoundingBoxLiteral;
import org.apache.iceberg.geospatial.BoundingBox;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transforms;
Expand Down Expand Up @@ -340,6 +342,8 @@ public <T> Expression predicate(UnboundPredicate<T> pred) {
case NOT_EQ:
case STARTS_WITH:
case NOT_STARTS_WITH:
case ST_INTERSECTS:
case ST_DISJOINT:
return new UnboundPredicate<>(
pred.op(), pred.term(), (T) sanitize(pred.literal(), now, today));
case IN:
Expand Down Expand Up @@ -440,6 +444,10 @@ public <T> String predicate(BoundPredicate<T> pred) {
return term + " STARTS WITH " + value((BoundLiteralPredicate<?>) pred);
case NOT_STARTS_WITH:
return term + " NOT STARTS WITH " + value((BoundLiteralPredicate<?>) pred);
case ST_INTERSECTS:
return "st_intersects(" + term + ", " + value((BoundLiteralPredicate<?>) pred) + ")";
case ST_DISJOINT:
return "st_disjoint(" + term + ", " + value((BoundLiteralPredicate<?>) pred) + ")";
default:
throw new UnsupportedOperationException(
"Cannot sanitize unsupported predicate type: " + pred.op());
Expand Down Expand Up @@ -492,6 +500,10 @@ public <T> String predicate(UnboundPredicate<T> pred) {
return term + " STARTS WITH " + sanitize(pred.literal(), nowMicros, today);
case NOT_STARTS_WITH:
return term + " NOT STARTS WITH " + sanitize(pred.literal(), nowMicros, today);
case ST_INTERSECTS:
return "st_intersects(" + term + ", " + sanitize(pred.literal(), nowMicros, today) + ")";
case ST_DISJOINT:
return "st_disjoint(" + term + ", " + sanitize(pred.literal(), nowMicros, today) + ")";
default:
throw new UnsupportedOperationException(
"Cannot sanitize unsupported predicate type: " + pred.op());
Expand Down Expand Up @@ -541,6 +553,8 @@ private static String sanitize(Literal<?> literal, long now, int today) {
return sanitizeNumber(((Literals.DoubleLiteral) literal).value(), "float");
} else if (literal instanceof Literals.VariantLiteral) {
return sanitizeVariant(((Literals.VariantLiteral) literal).value(), now, today);
} else if (literal instanceof BoundingBoxLiteral) {
return sanitizeBoundingBox(((BoundingBoxLiteral) literal).value());
} else {
// for uuid, decimal, fixed and binary, match the string result
return sanitizeSimpleString(literal.value().toString());
Expand Down Expand Up @@ -619,6 +633,20 @@ private static String sanitizeSimpleString(CharSequence value) {
return String.format(Locale.ROOT, "(hash-%08x)", HASH_FUNC.apply(value));
}

private static String sanitizeBoundingBox(BoundingBox bbox) {
boolean hasZ = bbox.min().hasZ() && bbox.max().hasZ();
boolean hasM = bbox.min().hasM() && bbox.max().hasM();
if (hasZ && hasM) {
return "(boundingbox-xyzm)";
} else if (hasZ) {
return "(boundingbox-xyz)";
} else if (hasM) {
return "(boundingbox-xym)";
} else {
return "(boundingbox-xy)";
}
}

private static String sanitizeVariant(Variant value, long now, int today) {
return sanitizeVariant(value.value(), now, today);
}
Expand Down
48 changes: 48 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/ExpressionVisitors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Set;
import java.util.function.Supplier;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.geospatial.BoundingBox;

/** Utils for traversing {@link Expression expressions}. */
public class ExpressionVisitors {
Expand Down Expand Up @@ -126,6 +127,14 @@ public <T> R notStartsWith(BoundReference<T> ref, Literal<T> lit) {
"notStartsWith expression is not supported by the visitor");
}

public <T> R stIntersects(BoundReference<T> ref, BoundingBox bbox) {
throw new UnsupportedOperationException("stIntersects expression is not supported");
}

public <T> R stDisjoint(BoundReference<T> ref, BoundingBox bbox) {
throw new UnsupportedOperationException("stDisjoint expression is not supported");
}

/**
* Handle a non-reference value in this visitor.
*
Expand All @@ -141,6 +150,7 @@ public <T> R handleNonReference(Bound<T> term) {
throw new ValidationException("Visitor %s does not support non-reference: %s", this, term);
}

@SuppressWarnings("unchecked")
@Override
public <T> R predicate(BoundPredicate<T> pred) {
if (!(pred.term() instanceof BoundReference)) {
Expand All @@ -166,6 +176,14 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith((BoundReference<T>) pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith((BoundReference<T>) pred.term(), literalPred.literal());
case ST_INTERSECTS:
return stIntersects(
(BoundReference<T>) pred.term(),
((Literals.BoundingBoxLiteral) literalPred.literal()).value());
case ST_DISJOINT:
return stDisjoint(
(BoundReference<T>) pred.term(),
((Literals.BoundingBoxLiteral) literalPred.literal()).value());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -266,6 +284,16 @@ public <T> R notStartsWith(Bound<T> expr, Literal<T> lit) {
throw new UnsupportedOperationException("Unsupported operation.");
}

public <T> R stIntersects(Bound<T> term, BoundingBox bbox) {
throw new UnsupportedOperationException(
"stIntersects operator is not supported by the visitor");
}

public <T> R stDisjoint(Bound<T> term, BoundingBox bbox) {
throw new UnsupportedOperationException(
"stDisjoint operator is not supported by the visitor");
}

@Override
public <T> R predicate(BoundPredicate<T> pred) {
if (pred.isLiteralPredicate()) {
Expand All @@ -287,6 +315,12 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith(pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith(pred.term(), literalPred.literal());
case ST_INTERSECTS:
return stIntersects(
pred.term(), ((Literals.BoundingBoxLiteral) literalPred.literal()).value());
case ST_DISJOINT:
return stDisjoint(
pred.term(), ((Literals.BoundingBoxLiteral) literalPred.literal()).value());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -465,6 +499,12 @@ public <T> R predicate(BoundPredicate<T> pred) {
return startsWith(pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith(pred.term(), literalPred.literal());
case ST_INTERSECTS:
return stIntersects(
pred.term(), ((Literals.BoundingBoxLiteral) literalPred.literal()).value());
case ST_DISJOINT:
return stDisjoint(
pred.term(), ((Literals.BoundingBoxLiteral) literalPred.literal()).value());
default:
throw new IllegalStateException(
"Invalid operation for BoundLiteralPredicate: " + pred.op());
Expand Down Expand Up @@ -555,6 +595,14 @@ public <T> R startsWith(BoundTerm<T> term, Literal<T> lit) {
public <T> R notStartsWith(BoundTerm<T> term, Literal<T> lit) {
return null;
}

public <T> R stIntersects(BoundTerm<T> term, BoundingBox bbox) {
return null;
}

public <T> R stDisjoint(BoundTerm<T> term, BoundingBox bbox) {
return null;
}
}

/**
Expand Down
17 changes: 17 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Expressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
*/
package org.apache.iceberg.expressions;

import java.nio.ByteBuffer;
import java.util.stream.Stream;
import org.apache.iceberg.expressions.Expression.Operation;
import org.apache.iceberg.geospatial.BoundingBox;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transform;
Expand Down Expand Up @@ -202,6 +204,14 @@ public static UnboundPredicate<String> notStartsWith(UnboundTerm<String> expr, S
return new UnboundPredicate<>(Expression.Operation.NOT_STARTS_WITH, expr, value);
}

public static UnboundPredicate<ByteBuffer> stIntersects(String name, BoundingBox value) {
Copy link
Contributor

@rdblue rdblue Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this a BoundingBox rather than passing values directly? Should we have a version that accepts xmin, xmax, ymin, ymax for simple cases?

Copy link
Contributor

@stevenzwu stevenzwu Feb 20, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GeospatialBound also have additionally optional dimensions z and m. Hence, I thought BoundingBox(GeospatialBound min, GeospatialBound max) is probably good here.

If we want to provide some convenient constructor, should we do it in BoundingBox?

public static BoundingBox from(xmin, xmax, ymin, ymax)

Copy link
Contributor

@rdblue rdblue Feb 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I'm fine either way. We could include an override here that produces a bounding box. I agree with not having 6 and 8 argument versions of the method. And we will need to consider what to name the factory methods in BoundingBox because it can be ambiguous whether you're passing z or m through a 6-arg factory method.

return geospatialPredicate(Operation.ST_INTERSECTS, name, value);
}

public static UnboundPredicate<ByteBuffer> stDisjoint(String name, BoundingBox value) {
return geospatialPredicate(Operation.ST_DISJOINT, name, value);
}

public static <T> UnboundPredicate<T> in(String name, T... values) {
return predicate(Operation.IN, name, Lists.newArrayList(values));
}
Expand Down Expand Up @@ -280,6 +290,13 @@ public static <T> UnboundPredicate<T> predicate(Operation op, UnboundTerm<T> exp
return new UnboundPredicate<>(op, expr);
}

@SuppressWarnings("unchecked")
static UnboundPredicate<ByteBuffer> geospatialPredicate(
Operation op, String name, BoundingBox value) {
return new UnboundPredicate<ByteBuffer>(
op, ref(name), (Literal<ByteBuffer>) (Literal<?>) Literal.of(value));
}

public static True alwaysTrue() {
return True.INSTANCE;
}
Expand Down
5 changes: 5 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Literal.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.UUID;
import org.apache.iceberg.geospatial.BoundingBox;
import org.apache.iceberg.types.Type;

/**
Expand Down Expand Up @@ -71,6 +72,10 @@ static Literal<BigDecimal> of(BigDecimal value) {
return new Literals.DecimalLiteral(value);
}

static Literal<BoundingBox> of(BoundingBox value) {
return new Literals.BoundingBoxLiteral(value);
}

/** Returns the value wrapped by this literal. */
T value();

Expand Down
96 changes: 96 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/Literals.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import java.util.Locale;
import java.util.Objects;
import java.util.UUID;
import org.apache.iceberg.geospatial.BoundingBox;
import org.apache.iceberg.geospatial.GeospatialBound;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.io.BaseEncoding;
import org.apache.iceberg.types.Comparators;
Expand Down Expand Up @@ -87,6 +89,8 @@ static <T> Literal<T> from(T value) {
return (Literal<T>) new Literals.DecimalLiteral((BigDecimal) value);
} else if (value instanceof Variant) {
return (Literal<T>) new Literals.VariantLiteral((Variant) value);
} else if (value instanceof BoundingBox) {
return (Literal<T>) new Literals.BoundingBoxLiteral((BoundingBox) value);
}

throw new IllegalArgumentException(
Expand Down Expand Up @@ -747,4 +751,96 @@ public String toString() {
return "X'" + BASE16_ENCODING.encode(bytes) + "'";
}
}

static class BoundingBoxLiteral implements Literal<BoundingBox> {
private static final Comparator<BoundingBox> CMP =
Comparators.<BoundingBox>nullsFirst().thenComparing(BoundingBoxLiteral::compare);

private final BoundingBox value;

BoundingBoxLiteral(BoundingBox value) {
Preconditions.checkNotNull(value, "Bounding box value cannot be null");
this.value = value;
}

@Override
public BoundingBox value() {
return value;
}

@Override
public ByteBuffer toByteBuffer() {
return value.toByteBuffer();
}

@Override
@SuppressWarnings("unchecked")
public <T> Literal<T> to(Type type) {
if (type.typeId() != Type.TypeID.GEOMETRY && type.typeId() != Type.TypeID.GEOGRAPHY) {
return null;
}

return (Literal<T>) this;
}

@Override
public Comparator<BoundingBox> comparator() {
return CMP;
}

Object writeReplace() throws ObjectStreamException {
return new SerializationProxies.BoundingBoxLiteralProxy(toByteBuffer());
}

@Override
public String toString() {
return value().toString();
}

@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof BoundingBoxLiteral)) {
return false;
}

BoundingBoxLiteral that = (BoundingBoxLiteral) other;
return comparator().compare(value(), that.value()) == 0;
Copy link
Contributor

@rdblue rdblue Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this works, then the comparator is incorrect for values. This is comparing serialized bounding boxes.

}

@Override
public int hashCode() {
return Objects.hashCode(value());
}

private static int compare(BoundingBox left, BoundingBox right) {
int cmp = compare(left.min(), right.min());
if (cmp != 0) {
return cmp;
}

return compare(left.max(), right.max());
}

private static int compare(GeospatialBound left, GeospatialBound right) {
int cmp = Double.compare(left.x(), right.x());
if (cmp != 0) {
return cmp;
}

cmp = Double.compare(left.y(), right.y());
if (cmp != 0) {
return cmp;
}

cmp = Double.compare(left.z(), right.z());
if (cmp != 0) {
return cmp;
}

return Double.compare(left.m(), right.m());
}
}
}
17 changes: 17 additions & 0 deletions api/src/main/java/org/apache/iceberg/expressions/SerializationProxies.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.ObjectStreamException;
import java.io.Serializable;
import java.nio.ByteBuffer;
import org.apache.iceberg.geospatial.BoundingBox;

/**
* Stand-in classes for expression classes in Java Serialization.
Expand Down Expand Up @@ -81,4 +82,20 @@ protected byte[] bytes() {
return bytes;
}
}

static class BoundingBoxLiteralProxy implements Serializable {
private byte[] bytes;

/** Constructor for Java serialization. */
BoundingBoxLiteralProxy() {}

BoundingBoxLiteralProxy(ByteBuffer buffer) {
this.bytes = new byte[buffer.remaining()];
buffer.duplicate().get(this.bytes);
}

Object readResolve() throws ObjectStreamException {
return new Literals.BoundingBoxLiteral(BoundingBox.fromByteBuffer(ByteBuffer.wrap(bytes)));
}
}
}
Loading