From 28eb266801559f0b886e7589934e2c29d8289d14 Mon Sep 17 00:00:00 2001 From: Nikola Vasiljevic <35523348+nikokaoja@users.noreply.github.com> Date: Tue, 14 May 2024 16:34:21 +0200 Subject: [PATCH] Neat 242 allow arbitrary filters but raise warning that we do not support them (#451) * added raw filter * added check if raw filter can be actually parsed * enable export of raw filter to dms schema * change log, bump version * Linting and static code checks * upgrade code * upgrade code * upgrade code --------- Co-authored-by: nikokaoja --- Makefile | 2 +- cognite/neat/_version.py | 2 +- cognite/neat/rules/models/dms/_exporter.py | 2 +- cognite/neat/rules/models/dms/_rules.py | 4 +-- cognite/neat/rules/models/wrapped_entities.py | 36 +++++++++++++++++-- docs/CHANGELOG.md | 5 +++ pyproject.toml | 2 +- .../test_models/test_wrapped_entities.py | 36 ++++++++++++++++++- 8 files changed, 80 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index fae5191cc..6bc4d7619 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up -version="0.77.0" +version="0.77.1" run-explorer: @echo "Running explorer API server..." # open "http://localhost:8000/static/index.html" || true diff --git a/cognite/neat/_version.py b/cognite/neat/_version.py index e2133e5b4..e7697f644 100644 --- a/cognite/neat/_version.py +++ b/cognite/neat/_version.py @@ -1 +1 @@ -__version__ = "0.77.0" +__version__ = "0.77.1" diff --git a/cognite/neat/rules/models/dms/_exporter.py b/cognite/neat/rules/models/dms/_exporter.py index 33ac6e913..26d40a412 100644 --- a/cognite/neat/rules/models/dms/_exporter.py +++ b/cognite/neat/rules/models/dms/_exporter.py @@ -133,7 +133,6 @@ def _create_views_with_node_types( view_filter = self._create_view_filter(view, dms_view, data_model_type, dms_properties) view.filter = view_filter.as_dms_filter() - if isinstance(view_filter, NodeTypeFilter): unique_node_types.update(view_filter.nodes) if view.as_id() in parent_views: @@ -254,6 +253,7 @@ def _create_view_filter( dms_properties: list[DMSProperty], ) -> DMSFilter: selected_filter_name = (dms_view and dms_view.filter_ and dms_view.filter_.name) or "" + if dms_view and dms_view.filter_ and not dms_view.filter_.is_empty: # Has Explicit Filter, use it return dms_view.filter_ diff --git a/cognite/neat/rules/models/dms/_rules.py b/cognite/neat/rules/models/dms/_rules.py index bcae82b62..6d45d26fc 100644 --- a/cognite/neat/rules/models/dms/_rules.py +++ b/cognite/neat/rules/models/dms/_rules.py @@ -41,7 +41,7 @@ ViewEntityList, ViewPropertyEntity, ) -from cognite.neat.rules.models.wrapped_entities import HasDataFilter, NodeTypeFilter +from cognite.neat.rules.models.wrapped_entities import HasDataFilter, NodeTypeFilter, RawFilter from ._schema import DMSSchema @@ -257,7 +257,7 @@ class DMSView(SheetEntity): description: str | None = Field(alias="Description", default=None) implements: ViewEntityList | None = Field(None, alias="Implements") reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right") - filter_: HasDataFilter | NodeTypeFilter | None = Field(None, alias="Filter") + filter_: HasDataFilter | NodeTypeFilter | RawFilter | None = Field(None, alias="Filter") in_model: bool = Field(True, alias="In Model") class_: ClassEntity = Field(alias="Class (linage)") diff --git a/cognite/neat/rules/models/wrapped_entities.py b/cognite/neat/rules/models/wrapped_entities.py index 187d0e4c6..6f75c466a 100644 --- a/cognite/neat/rules/models/wrapped_entities.py +++ b/cognite/neat/rules/models/wrapped_entities.py @@ -1,3 +1,5 @@ +import json +import re from abc import ABC, abstractmethod from collections.abc import Collection from functools import total_ordering @@ -37,8 +39,19 @@ def _load(cls, data: Any) -> dict: def _parse(cls, data: str) -> dict: if data.casefold() == cls.name.casefold(): return {"inner": None} - inner = data[len(cls.name) :].removeprefix("(").removesuffix(")") - return {"inner": [cls._inner_cls.load(entry.strip()) for entry in inner.split(",")]} + + # raw filter case: + if cls.__name__ == "RawFilter": + if match := re.search(r"rawFilter\(([\s\S]*?)\)", data): + return {"filter": match.group(1), "inner": None} + else: + raise ValueError(f"Cannot parse {cls.name} from {data}. Ill formatted raw filter.") + + # nodeType and hasData case: + elif inner := data[len(cls.name) :].removeprefix("(").removesuffix(")"): + return {"inner": [cls._inner_cls.load(entry.strip()) for entry in inner.split(",")]} + else: + raise ValueError(f"Cannot parse {cls.name} from {data}") @model_serializer(when_used="unless-none", return_type=str) def as_str(self) -> str: @@ -164,3 +177,22 @@ def as_dms_filter(self, default: Collection[ContainerId] | None = None) -> dm.Fi # Sorting to ensure deterministic order containers=sorted(containers, key=lambda container: container.as_tuple()) # type: ignore[union-attr] ) + + +class RawFilter(DMSFilter): + name: ClassVar[str] = "rawFilter" + filter: str + inner: None = None # type: ignore[assignment] + + def as_dms_filter(self) -> dm.Filter: # type: ignore[override] + try: + return dm.Filter.load(json.loads(self.filter)) + except json.JSONDecodeError as e: + raise ValueError(f"Error loading raw filter: {e}") from e + + @property + def is_empty(self) -> bool: + return self.filter is None + + def __repr__(self) -> str: + return self.filter diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bf4eac43e..d6dd9ede9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,11 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [0.77.1] - 14-05-24 +### Added +- Support for `RawFilters` allow arbitrary filters to be applied to the data model. + + ## [0.77.0] - 13-05-24 ### Changed - [BREAKING] The subpackage `cognite.neat.rules.models` is reorganized. All imports using this subpackage must be diff --git a/pyproject.toml b/pyproject.toml index 382a61135..f617077b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite-neat" -version = "0.77.0" +version = "0.77.1" readme = "README.md" description = "Knowledge graph transformation" authors = [ diff --git a/tests/tests_unit/rules/test_models/test_wrapped_entities.py b/tests/tests_unit/rules/test_models/test_wrapped_entities.py index f9c040238..06445b29b 100644 --- a/tests/tests_unit/rules/test_models/test_wrapped_entities.py +++ b/tests/tests_unit/rules/test_models/test_wrapped_entities.py @@ -4,7 +4,24 @@ from cognite.client import data_modeling as dm from cognite.neat.rules.models.entities import ContainerEntity, DMSNodeEntity -from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter, WrappedEntity +from cognite.neat.rules.models.wrapped_entities import ( + DMSFilter, + HasDataFilter, + NodeTypeFilter, + RawFilter, + WrappedEntity, +) + +RAW_FILTER_EXAMPLE = """{"and": [ + { + "in": { + "property": ["yggdrasil_domain_model", "EntityTypeGroup", "entityType"], + "values": ["CFIHOS_00000003"] + } + } + ]}""" + +RAW_FILTER_CELL_EXAMPLE = f"""rawFilter({RAW_FILTER_EXAMPLE})""" class TestWrappedEntities: @@ -43,6 +60,11 @@ class TestWrappedEntities: ] ), ), + ( + RawFilter, + RAW_FILTER_CELL_EXAMPLE, + RawFilter(filter=RAW_FILTER_EXAMPLE), + ), ], ) def test_load(self, cls_: type[WrappedEntity], raw: Any, expected: WrappedEntity) -> None: @@ -81,3 +103,15 @@ def test_from_dms_filter(self, filter_: dm.Filter, expected: DMSFilter) -> None: loaded = DMSFilter.from_dms_filter(filter_) assert loaded == expected + + def test_has_data_vs_raw_filter(self) -> None: + assert ( + HasDataFilter.load("hasData(space:container1)").as_dms_filter().dump() + == RawFilter.load( + """rawFilter({"hasData": [{"type": "container", + "space": "space", + "externalId": "container1"}]})""" + ) + .as_dms_filter() + .dump() + )