From ba2b37df016ac0ebfcf177e25f00adf24244eb1c Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 13 Mar 2024 05:18:49 +0000 Subject: [PATCH] add NotType and NotExpression --- datacube/index/abstract.py | 12 ++++++------ datacube/index/fields.py | 14 +++++++++++++- datacube/model/__init__.py | 2 +- datacube/model/_base.py | 3 +++ integration_tests/index/test_config_docs.py | 12 +++++++++--- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 87b671cfe..3654eae0c 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -21,7 +21,7 @@ from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException from datacube.index.fields import Field -from datacube.model import Dataset, MetadataType, Range +from datacube.model import Dataset, MetadataType, Range, NotType from datacube.model import Product from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged @@ -374,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]: yield mdt.definition -QueryField = Union[str, float, int, Range, datetime.datetime] +QueryField = Union[str, float, int, Range, datetime.datetime, NotType] QueryDict = Mapping[str, QueryField] @@ -688,7 +688,7 @@ def search(self, **query: QueryField) -> Iterator[Product]: @abstractmethod def search_robust(self, **query: QueryField - ) -> Iterable[Tuple[Product, Mapping[str, QueryField]]]: + ) -> Iterable[Tuple[Product, QueryDict]]: """ Return dataset types that match match-able fields and dict of remaining un-matchable fields. @@ -698,7 +698,7 @@ def search_robust(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Product objects. @@ -1115,7 +1115,7 @@ def restore_location(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Dataset objects. @@ -1129,7 +1129,7 @@ def search_by_metadata(self, @abstractmethod def search(self, limit: Optional[int] = None, - source_filter: Optional[Mapping[str, QueryField]] = None, + source_filter: Optional[QueryDict] = None, **query: QueryField) -> Iterable[Dataset]: """ Perform a search, returning results as Dataset objects. diff --git a/datacube/index/fields.py b/datacube/index/fields.py index c67a1115d..557edf4e9 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -10,7 +10,7 @@ from dateutil.tz import tz from typing import List -from datacube.model import Range +from datacube.model import Range, NotType from datacube.model.fields import Expression, Field __all__ = ['Field', @@ -36,6 +36,16 @@ def evaluate(self, ctx): return any(expr.evaluate(ctx) for expr in self.exprs) +class NotExpression(Expression): + def __init__(self, expr): + super(NotExpression, self).__init__() + self.expr = expr + self.field = expr.field + + def evaluate(self, ctx): + return not self.expr.evaluate(ctx) + + def as_expression(field: Field, value) -> Expression: """ Convert a single field/value to expression, following the "simple" conventions. @@ -44,6 +54,8 @@ def as_expression(field: Field, value) -> Expression: return field.between(value.begin, value.end) elif isinstance(value, list): return OrExpression(*(as_expression(field, val) for val in value)) + elif isinstance(value, NotType): + return NotExpression(as_expression(field, value.value)) # Treat a date (day) as a time range. elif isinstance(value, date) and not isinstance(value, datetime): return as_expression( diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 1880b3345..14a346a47 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -21,7 +21,7 @@ schema_validated, DocReader from datacube.index.eo3 import is_doc_eo3 from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap # noqa: F401 +from ._base import Range, ranges_overlap, NotType # noqa: F401 from .eo3 import validate_eo3_compatible_type from deprecat import deprecat diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 103765866..9f5ca5f48 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -18,3 +18,6 @@ def ranges_overlap(ra: Range, rb: Range) -> bool: if ra.begin <= rb.begin: return ra.end > rb.begin return rb.end > ra.begin + + +NotType = namedtuple('NotType', 'value') diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index a5483dc1c..a4913e461 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -15,7 +15,7 @@ from datacube.index import Index from datacube.index.abstract import default_metadata_type_docs from datacube.model import MetadataType, DatasetType -from datacube.model import Range, Dataset +from datacube.model import Range, NotType, Dataset from datacube.utils import changes from datacube.utils.documents import documents_equal from datacube.testutils import sanitise_doc @@ -447,7 +447,7 @@ def test_filter_types_by_fields(index, wo_eo3_product): assert len(res) == 0 -def test_filter_types_by_search(index, wo_eo3_product): +def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product): """ :type ls5_telem_type: datacube.model.DatasetType :type index: datacube.index.Index @@ -456,7 +456,7 @@ def test_filter_types_by_search(index, wo_eo3_product): # No arguments, return all. res = list(index.products.search()) - assert res == [wo_eo3_product] + assert res == [ls8_eo3_product, wo_eo3_product] # Matching fields res = list(index.products.search( @@ -491,6 +491,12 @@ def test_filter_types_by_search(index, wo_eo3_product): )) assert res == [wo_eo3_product] + # Not expression test + res = list(index.products.search( + product_family=NotType("wo"), + )) + assert res == [ls8_eo3_product] + # Mismatching fields res = list(index.products.search( product_family='spam',