Custom queryables support

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.57.0] - 2026-07-27
+
+### Added
+
+- `tilebox-datasets`: Added typed server-side dataset query filters with comparison, logical, and null-check
+  expressions through `field()` and the `filter` query argument.
+
+### Fixed
+
+- `tilebox-datasets`: Preserve unsupported protobuf message fields, including STAC well-known types, as object-valued
+  xarray variables instead of failing dataset queries.
+
 ## [0.56.0] - 2026-07-16
 
 ### Added
@@ -417,7 +429,8 @@ the first client that does not cache data (since it's already on the local file
 - Released under the [MIT](https://opensource.org/license/mit) license.
 - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc`
 
-[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.56.0...HEAD
+[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.57.0...HEAD
+[0.57.0]: https://github.com/tilebox/tilebox-python/compare/v0.56.0...v0.57.0
 [0.56.0]: https://github.com/tilebox/tilebox-python/compare/v0.55.1...v0.56.0
 [0.55.1]: https://github.com/tilebox/tilebox-python/compare/v0.55.0...v0.55.1
 [0.55.0]: https://github.com/tilebox/tilebox-python/compare/v0.54.0...v0.55.0

README.md

@@ -65,15 +65,15 @@ sentinel2_msi = client.dataset("open_data.copernicus.sentinel2_msi")
 collections = sentinel2_msi.collections()
 print(collections)
 
-area_of_interest = shape({
-    "type": "Polygon",  # coords in lon, lat
-    "coordinates": [[[-5, 50], [-5, 56], [-11, 56], [-11, 50], [-5, 50]]]}
+area_of_interest = shape(
+    {
+        "type": "Polygon",  # coords in lon, lat
+        "coordinates": [[[-5, 50], [-5, 56], [-11, 56], [-11, 50], [-5, 50]]],
+    }
 )
 s2a_l1c = sentinel2_msi.collection("S2A_S2MSI1C")
 results = s2a_l1c.query(
-  temporal_extent=("2025-03-01", "2025-06-01"),
-  spatial_extent=area_of_interest,
-  show_progress=True
+    temporal_extent=("2025-03-01", "2025-06-01"), spatial_extent=area_of_interest, show_progress=True
 )
 print(f"Found {results.sizes['time']} datapoints")  # Found 979 datapoints
 ```
@@ -85,9 +85,10 @@ A parallel processing engine to simplify the creation of dynamic tasks that can
 ```python
 from tilebox.workflows import Client, Task
 
+
 class MyFirstTask(Task):
-  def execute(self):
-    print("Hello World from my first Tilebox task!")
+    def execute(self):
+        print("Hello World from my first Tilebox task!")
 
 
 # create your API key at https://console.tilebox.com

prek.toml

@@ -0,0 +1,45 @@
+# Configuration file for `prek`, a git hook framework written in Rust.
+# See https://prek.j178.dev for more information.
+#:schema https://www.schemastore.org/prek.json
+
+[[repos]]
+repo = "https://github.com/pre-commit/pre-commit-hooks"
+rev = "v6.0.0"
+hooks = [
+  { id = "check-yaml" },
+  { id = "end-of-file-fixer" }
+]
+
+[[repos]]
+repo = "https://github.com/tsvikas/sync-with-uv"
+rev = "v0.6.0"
+hooks = [
+  { id = "sync-with-uv" }
+]
+
+[[repos]]
+repo = "https://github.com/charliermarsh/ruff-pre-commit"
+rev = "v0.16.0"
+hooks = [
+  {
+    id = "ruff-check",
+    args = [
+      "--fix",
+      "--exit-non-zero-on-fix"
+    ]
+  },
+  { id = "ruff-format" }
+]
+
+[[repos]]
+repo = "local"
+hooks = [
+  {
+    id = "ty",
+    name = "ty-check",
+    entry = "uv run ty check",
+    language = "python",
+    types = ["python"],
+    pass_filenames = true
+  }
+]

pyproject.toml

@@ -94,6 +94,7 @@ ignore = [
     "PLR2004", # magic-value-comparison: sometimes comparison with constants (e.g. 0) makes sense
     "TRY003",  # raise-vanilla-args: exceptions like this make sense in python
     "TRY400",  # error-instead-of-exception: logger.error is ok with loguru
+    "CPY001",  # missing-copyright-notice: we don't add copyright notices to our source files
     # disabled because of formatter
     "E501",   # line-too-long -> formatter takes care of this
     "ISC001", # single-line-implicit-str-concatenation -> formatter takes care of this

tilebox-datasets/README.md

@@ -65,10 +65,7 @@ Query data:
 
 ```python
 s2a_l1c = sentinel2_msi.collection("S2A_S2MSI1C")
-results = s2a_l1c.query(
-  temporal_extent=("2025-03-01", "2025-06-01"),
-  show_progress=True
-)
+results = s2a_l1c.query(temporal_extent=("2025-03-01", "2025-06-01"), show_progress=True)
 print(f"Found {results.sizes['time']} datapoints")  # Found 220542 datapoints
 ```
 
@@ -77,15 +74,15 @@ Spatio-temporal queries:
 ```python
 from shapely.geometry import shape
 
-area_of_interest = shape({
-    "type": "Polygon",  # coords in lon, lat
-    "coordinates": [[[-5, 50], [-5, 56], [-11, 56], [-11, 50], [-5, 50]]]}
+area_of_interest = shape(
+    {
+        "type": "Polygon",  # coords in lon, lat
+        "coordinates": [[[-5, 50], [-5, 56], [-11, 56], [-11, 50], [-5, 50]]],
+    }
 )
 s2a_l1c = sentinel2_msi.collection("S2A_S2MSI1C")
 results = s2a_l1c.query(
-  temporal_extent=("2025-03-01", "2025-06-01"),
-  spatial_extent=area_of_interest,
-  show_progress=True
+    temporal_extent=("2025-03-01", "2025-06-01"), spatial_extent=area_of_interest, show_progress=True
 )
 print(f"Found {results.sizes['time']} datapoints")  # Found 979 datapoints
 ```

tilebox-datasets/tests/data/test_data_access.py

@@ -1,8 +1,15 @@
+from datetime import datetime, timedelta, timezone
+from uuid import uuid4
+
+import pytest
 from hypothesis import given
 from shapely import Geometry
 
 from tests.data.data_access import query_filters, spatial_filter_likes, spatial_filters
 from tilebox.datasets.data.data_access import QueryFilters, SpatialFilter, SpatialFilterDict
+from tilebox.datasets.datasets.v1 import data_access_pb2
+from tilebox.datasets.query import TimeInterval, field
+from tilebox.datasets.query.id_interval import IDInterval
 
 
 @given(spatial_filters())
@@ -33,3 +40,64 @@ def test_parse_spatial_filter_like(spatial_filter_like: Geometry | SpatialFilter
 @given(query_filters())
 def test_query_filters_to_message_and_back(q: QueryFilters) -> None:
     assert QueryFilters.from_message(q.to_message()) == q
+
+
+def test_query_filters_expression_to_message_and_back() -> None:
+    query_filter = QueryFilters(
+        temporal_extent=TimeInterval(
+            datetime(2026, 7, 27, tzinfo=timezone.utc),
+            datetime(2026, 7, 28, tzinfo=timezone.utc),
+        ),
+        filter=(field("cloud_cover") < 20) | field("cloud_cover").is_null(),
+    )
+
+    message = query_filter.to_message()
+    assert len(message.expressions) == 1
+    assert QueryFilters.from_message(message) == query_filter
+
+
+def test_multiple_wire_expressions_are_combined_with_and() -> None:
+    query_filter = QueryFilters(
+        TimeInterval(
+            datetime.now(tz=timezone.utc),
+            datetime.now(tz=timezone.utc) + timedelta(days=1),
+        )
+    )
+    message = query_filter.to_message()
+    message.expressions.extend(
+        [
+            (field("cloud_cover") < 20).to_message(),
+            (field("quality") >= 80).to_message(),
+        ]
+    )
+
+    round_tripped = QueryFilters.from_message(message).to_message()
+    assert len(round_tripped.expressions) == 1
+    assert round_tripped.expressions[0].logical.operator == data_access_pb2.LOGICAL_OPERATOR_AND
+    assert len(round_tripped.expressions[0].logical.operands) == 2
+
+
+def test_query_filters_reject_invalid_filter() -> None:
+    with pytest.raises(TypeError, match="Expected a query expression"):
+        QueryFilters(
+            TimeInterval(
+                datetime(2026, 7, 27, tzinfo=timezone.utc),
+                datetime(2026, 7, 28, tzinfo=timezone.utc),
+            ),
+            filter="quality > 80",  # type: ignore[arg-type]
+        )
+
+
+def test_query_filters_reject_invalid_interval_variants() -> None:
+    with pytest.raises(ValueError, match="exactly one time or datapoint interval"):
+        QueryFilters.from_message(data_access_pb2.QueryFilters())
+
+    message = QueryFilters(
+        TimeInterval(
+            datetime(2026, 7, 27, tzinfo=timezone.utc),
+            datetime(2026, 7, 28, tzinfo=timezone.utc),
+        )
+    ).to_message()
+    message.datapoint_interval.CopyFrom(IDInterval(uuid4(), uuid4(), False, False).to_message())
+    with pytest.raises(ValueError, match="exactly one time or datapoint interval"):
+        QueryFilters.from_message(message)

tilebox-datasets/tests/data/test_datasets.py

@@ -117,10 +117,10 @@ def test_create_and_update_dataset_include_field_annotations(operation: str) ->
     annotations = {field.descriptor.name: field.annotation for field in request.type.fields}
     assert annotations["time"].source_json_pointer == "/properties/datetime"
     assert annotations["time"].json_schema_ref.endswith("datetime.json#/properties/datetime")
-    assert annotations["id"].source_json_pointer == "/properties/tilebox_id"
+    assert annotations["id"].source_json_pointer == "/properties/tilebox:id"
     assert not annotations["id"].HasField("json_schema_ref")
-    assert annotations["ingestion_time"].source_json_pointer == "/properties/created"
-    assert annotations["ingestion_time"].json_schema_ref.endswith("datetime.json#/properties/created")
+    assert annotations["ingestion_time"].source_json_pointer == "/properties/tilebox:ingestion_time"
+    assert not annotations["ingestion_time"].HasField("json_schema_ref")
     assert annotations["geometry"].source_json_pointer == "/geometry"
     assert annotations["geometry"].json_schema_ref == "https://geojson.org/schema/Geometry.json"
     assert all(

tilebox-datasets/tests/protobuf_conversion/test_protobuf_xarray.py

@@ -2,6 +2,9 @@
 
 import pandas as pd
 import pytest
+from google.protobuf import descriptor_pb2
+from google.protobuf.descriptor_pool import Default
+from google.protobuf.message_factory import GetMessageClass
 from hypothesis import given, settings
 from hypothesis.strategies import lists
 from numpy.testing import assert_array_almost_equal, assert_array_equal
@@ -11,6 +14,7 @@
 
 from tests.data.datapoint import example_datapoints
 from tests.example_dataset.example_dataset_pb2 import ExampleDatapoint
+from tilebox.datasets.datasets.stac.v1.asset_pb2 import Assets
 from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter
 from tilebox.datasets.query.time_interval import timestamp_to_datetime, us_to_datetime
 
@@ -99,6 +103,45 @@ def test_convert_datapoint(datapoint: ExampleDatapoint) -> None:  # noqa: PLR091
         assert isinstance(dataset.some_repeated_geometry[i].item(), Polygon | MultiPolygon)
 
 
+def test_convert_unknown_message_fields_as_objects() -> None:
+    file_descriptor = descriptor_pb2.FileDescriptorProto(
+        name="tests/protobuf_conversion/stac_datapoint.proto",
+        package="tests.protobuf_conversion",
+        dependency=["datasets/stac/v1/asset.proto"],
+    )
+    message_descriptor = file_descriptor.message_type.add(name="StacDatapoint")
+    message_descriptor.field.add(
+        name="assets",
+        number=1,
+        label=descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL,
+        type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=".datasets.stac.v1.Assets",
+    )
+    message_descriptor.field.add(
+        name="related_assets",
+        number=2,
+        label=descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED,
+        type=descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=".datasets.stac.v1.Assets",
+    )
+    descriptor = Default().AddSerializedFile(file_descriptor.SerializeToString())
+    message_type = GetMessageClass(descriptor.message_types_by_name["StacDatapoint"])
+
+    assets = Assets()
+    related_assets = Assets()
+    messages = [message_type(), message_type(assets=assets, related_assets=[related_assets])]
+
+    converter = MessageToXarrayConverter()
+    converter.convert_all(messages)
+    dataset = converter.finalize("time")
+
+    assert dataset.assets.dtype == object
+    assert dataset.assets[0].item() is None
+    assert dataset.assets[1].item() == assets
+    assert dataset.related_assets.dtype == object
+    assert dataset.related_assets[1, 0].item() == related_assets
+
+
 @given(lists(example_datapoints(generated_fields=True, missing_fields=True), min_size=5, max_size=30))
 def test_convert_datapoints(datapoints: list[ExampleDatapoint]) -> None:  # noqa: C901, PLR0912
     converter = MessageToXarrayConverter()