Merge pull request #1563 from opendatacube/not_expression

add "Not" operator

datacube/index/abstract.py

@@ -21,7 +21,7 @@
 from datacube.config import LocalConfig
 from datacube.index.exceptions import TransactionException
 from datacube.index.fields import Field
-from datacube.model import Dataset, MetadataType, Range
+from datacube.model import Dataset, MetadataType, Range, Not
 from datacube.model import Product
 from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException
 from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged
@@ -374,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]:
             yield mdt.definition
 
 
-QueryField = Union[str, float, int, Range, datetime.datetime]
+QueryField = Union[str, float, int, Range, datetime.datetime, Not]
 QueryDict = Mapping[str, QueryField]
 
 
@@ -688,7 +688,7 @@ def search(self, **query: QueryField) -> Iterator[Product]:
     @abstractmethod
     def search_robust(self,
                       **query: QueryField
-                     ) -> Iterable[Tuple[Product, Mapping[str, QueryField]]]:
+                     ) -> Iterable[Tuple[Product, QueryDict]]:
         """
         Return dataset types that match match-able fields and dict of remaining un-matchable fields.
 
@@ -698,7 +698,7 @@ def search_robust(self,
 
     @abstractmethod
     def search_by_metadata(self,
-                           metadata: Mapping[str, QueryField]
+                           metadata: QueryDict
                            ) -> Iterable[Dataset]:
         """
         Perform a search using arbitrary metadata, returning results as Product objects.
@@ -1115,7 +1115,7 @@ def restore_location(self,
 
     @abstractmethod
     def search_by_metadata(self,
-                           metadata: Mapping[str, QueryField]
+                           metadata: QueryDict
                           ) -> Iterable[Dataset]:
         """
         Perform a search using arbitrary metadata, returning results as Dataset objects.
@@ -1129,7 +1129,7 @@ def search_by_metadata(self,
     @abstractmethod
     def search(self,
                limit: Optional[int] = None,
-               source_filter: Optional[Mapping[str, QueryField]] = None,
+               source_filter: Optional[QueryDict] = None,
                **query: QueryField) -> Iterable[Dataset]:
         """
         Perform a search, returning results as Dataset objects.

datacube/index/fields.py

@@ -10,7 +10,7 @@
 from dateutil.tz import tz
 from typing import List
 
-from datacube.model import Range
+from datacube.model import Range, Not
 from datacube.model.fields import Expression, Field
 
 __all__ = ['Field',
@@ -36,6 +36,16 @@ def evaluate(self, ctx):
         return any(expr.evaluate(ctx) for expr in self.exprs)
 
 
+class NotExpression(Expression):
+    def __init__(self, expr):
+        super(NotExpression, self).__init__()
+        self.expr = expr
+        self.field = expr.field
+
+    def evaluate(self, ctx):
+        return not self.expr.evaluate(ctx)
+
+
 def as_expression(field: Field, value) -> Expression:
     """
     Convert a single field/value to expression, following the "simple" conventions.
@@ -44,6 +54,8 @@ def as_expression(field: Field, value) -> Expression:
         return field.between(value.begin, value.end)
     elif isinstance(value, list):
         return OrExpression(*(as_expression(field, val) for val in value))
+    elif isinstance(value, Not):
+        return NotExpression(as_expression(field, value.value))
     # Treat a date (day) as a time range.
     elif isinstance(value, date) and not isinstance(value, datetime):
         return as_expression(

datacube/model/__init__.py

@@ -21,7 +21,7 @@
     schema_validated, DocReader
 from datacube.index.eo3 import is_doc_eo3
 from .fields import Field, get_dataset_fields
-from ._base import Range, ranges_overlap  # noqa: F401
+from ._base import Range, ranges_overlap, Not  # noqa: F401
 from .eo3 import validate_eo3_compatible_type
 
 from deprecat import deprecat

datacube/model/_base.py

@@ -18,3 +18,6 @@ def ranges_overlap(ra: Range, rb: Range) -> bool:
     if ra.begin <= rb.begin:
         return ra.end > rb.begin
     return rb.end > ra.begin
+
+
+Not = namedtuple('Not', 'value')

docs/about/whats_new.rst

@@ -16,6 +16,7 @@ v1.8.next
 - Tweak ``list_products`` logic for getting crs and resolution values (:pull:`1535`)
 - Add new ODC Cheatsheet reference doc to Data Access & Analysis documentation page (:pull:`1543`)
 - Fix broken codecov github action. (:pull:`1554`)
+- Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`)
 
 v1.8.17 (8th November 2023)
 ===========================

integration_tests/index/test_config_docs.py

@@ -15,7 +15,7 @@
 from datacube.index import Index
 from datacube.index.abstract import default_metadata_type_docs
 from datacube.model import MetadataType, DatasetType
-from datacube.model import Range, Dataset
+from datacube.model import Range, Not, Dataset
 from datacube.utils import changes
 from datacube.utils.documents import documents_equal
 from datacube.testutils import sanitise_doc
@@ -447,7 +447,7 @@ def test_filter_types_by_fields(index, wo_eo3_product):
     assert len(res) == 0
 
 
-def test_filter_types_by_search(index, wo_eo3_product):
+def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product):
     """
     :type ls5_telem_type: datacube.model.DatasetType
     :type index: datacube.index.Index
@@ -456,7 +456,7 @@ def test_filter_types_by_search(index, wo_eo3_product):
 
     # No arguments, return all.
     res = list(index.products.search())
-    assert res == [wo_eo3_product]
+    assert res == [ls8_eo3_product, wo_eo3_product]
 
     # Matching fields
     res = list(index.products.search(
@@ -491,6 +491,12 @@ def test_filter_types_by_search(index, wo_eo3_product):
     ))
     assert res == [wo_eo3_product]
 
+    # Not expression test
+    res = list(index.products.search(
+        product_family=Not("wo"),
+    ))
+    assert res == [ls8_eo3_product]
+
     # Mismatching fields
     res = list(index.products.search(
         product_family='spam',