Handle Unknown and Invalid filters (#1722)

CHANGELOG.md

@@ -17,6 +17,11 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [7.36.0] - 2024-04-16
+### Fixed
+- Now handle unknown filter types
+- Add support for the "invalid" filter type in DMS
+
 ## [7.35.0] - 2024-04-16
 ### Added
 - Datapoints insert methods `insert` and `insert_multiple` now support ingesting (optional) status codes.

cognite/client/_version.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "7.35.0"
+__version__ = "7.36.0"
 __api_subversion__ = "20230101"

cognite/client/data_classes/filters.py

@@ -8,7 +8,7 @@
 
 from cognite.client.data_classes._base import EnumProperty, Geometry
 from cognite.client.data_classes.labels import Label
-from cognite.client.utils._text import to_camel_case
+from cognite.client.utils._text import convert_all_keys_to_camel_case, to_camel_case
 from cognite.client.utils.useful_types import SequenceNotStr
 
 if TYPE_CHECKING:
@@ -185,8 +185,13 @@ def load(cls, filter_: dict[str, Any]) -> Filter:
                 property=filter_body["property"],
                 value=_load_filter_value(filter_body["value"]),
             )
+        elif filter_name == InvalidFilter._filter_name:
+            return InvalidFilter(
+                previously_referenced_properties=filter_body["previouslyReferencedProperties"],
+                filter_type=filter_body["filterType"],
+            )
         else:
-            raise ValueError(f"Unknown filter type: {filter_name}")
+            return UnknownFilter(filter_name, filter_body)
 
     @abstractmethod
     def _filter_body(self, camel_case_property: bool) -> list | dict: ...
@@ -199,6 +204,42 @@ def _involved_filter_types(self) -> set[type[Filter]]:
         return output
 
 
+class UnknownFilter(Filter):
+    def __init__(self, filter_name: str, filter_body: dict[str, Any]) -> None:
+        self.__actual_filter_name = filter_name
+        self.__actual_filter_body = filter_body
+
+    def _filter_body(self, camel_case_property: bool) -> dict[str, Any]:
+        if camel_case_property:
+            return convert_all_keys_to_camel_case(self.__actual_filter_body)
+        else:
+            return self.__actual_filter_body
+
+    def dump(self, camel_case_property: bool = False) -> dict[str, Any]:
+        return {self.__actual_filter_name: self._filter_body(camel_case_property)}
+
+
+class InvalidFilter(Filter):
+    _filter_name = "invalid"
+
+    def __init__(self, previously_referenced_properties: list[list[str]], filter_type: str) -> None:
+        self.__previously_reference_properties = previously_referenced_properties
+        self.__filter_type = filter_type
+
+    def _filter_body(self, camel_case_property: bool) -> dict[str, Any]:
+        body = {
+            "previously_referenced_properties": self.__previously_reference_properties,
+            "filter_type": self.__filter_type,
+        }
+        if camel_case_property:
+            return convert_all_keys_to_camel_case(body)
+        else:
+            return body
+
+    def dump(self, camel_case_property: bool = False) -> dict[str, Any]:
+        return {self._filter_name: self._filter_body(camel_case_property)}
+
+
 def _validate_filter(
     filter: Filter | dict[str, Any] | None, supported_filters: frozenset[type[Filter]], api_name: str
 ) -> None:

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "cognite-sdk"
 
-version = "7.35.0"
+version = "7.36.0"
 description = "Cognite Python SDK"
 readme = "README.md"
 documentation = "https://cognite-sdk-python.readthedocs-hosted.com"

tests/tests_unit/test_data_classes/test_data_models/test_filters.py

@@ -176,6 +176,11 @@ def dump_filter_test_data() -> Iterator[ParameterSet]:
     }
     yield pytest.param(snake_cased_property, expected, id="And range filter with snake cased property")
 
+    yield pytest.param(
+        f.InvalidFilter([["some", "old", "prop"]], "overlaps"),
+        {"invalid": {"previously_referenced_properties": [["some", "old", "prop"]], "filter_type": "overlaps"}},
+    )
+
 
 @pytest.mark.parametrize("user_filter, expected", list(dump_filter_test_data()))
 def test_dump_filter(user_filter: Filter, expected: dict) -> None:
@@ -185,8 +190,9 @@ def test_dump_filter(user_filter: Filter, expected: dict) -> None:
 
 
 def test_unknown_filter_type() -> None:
-    with pytest.raises(ValueError, match="Unknown filter type: unknown"):
-        Filter.load({"unknown": {}})
+    unknown = Filter.load({"unknown": {}})
+    assert isinstance(unknown, f.UnknownFilter)
+    assert unknown.dump() == {"unknown": {}}
 
 
 @pytest.mark.parametrize("property_cls", filter(lambda cls: hasattr(cls, "metadata_key"), all_subclasses(EnumProperty)))

tests/utils.py

@@ -430,8 +430,12 @@ def create_value(self, type_: Any, var_name: str | None = None) -> Any:
         elif inspect.isclass(type_) and any(base is abc.ABC for base in type_.__bases__):
             implementations = all_concrete_subclasses(type_)
             if type_ is Filter:
+                # Remove filters not supported by dps subscriptions
+                implementations.remove(filters.Overlaps)
+
                 # Remove filters which are only used by data modeling classes
                 implementations.remove(filters.HasData)
+                implementations.remove(filters.InvalidFilter)
                 implementations.remove(filters.Nested)
                 implementations.remove(filters.GeoJSONWithin)
                 implementations.remove(filters.GeoJSONDisjoint)