[NEAT-217] Import/Export Unknown Type (#422)

* fix: bump + changelog

* build: forgotten

* tests: Added failing test

* fix: load unknown correctly

* tests: updated test to failing

* fix: implemented correct conversion as well

* Update cognite/neat/graph/extractors/_mock_graph_generator.py

* Update cognite/neat/graph/extractors/_mock_graph_generator.py

* tests: upgraded tests

* refactor: cleanup

* tests: Added test for DMS source as well

* refactor: improved error message

* tests: fixed missing

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.75.7"
+version="0.75.8"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.75.7"
+__version__ = "0.75.8"

cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py

@@ -1,5 +1,6 @@
 from collections import Counter
 from collections.abc import Callable, Sequence
+from typing import cast
 
 import cognite.neat.rules.issues.importing
 from cognite.neat.rules import issues
@@ -88,7 +89,12 @@ def convert_interface(self, item: Interface, _: str | None) -> None:
             name=item.display_name,
             description=item.description,
             comment=item.comment,
-            parent=[ParentClassEntity.load(parent.as_class_id()) for parent in item.extends or []] or None,
+            parent=[
+                cast(ParentClassEntity, parent_entity)
+                for parent in item.extends or []
+                if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
+            ]
+            or None,
         )
         self.classes.append(class_)
         for sub_item_or_id in item.contents or []:

cognite/neat/rules/issues/dms.py

@@ -195,7 +195,7 @@ class DirectRelationMissingSourceWarning(DMSSchemaWarning):
     property: str
 
     def message(self) -> str:
-        return f"The source view referred to by {self.view_id}.{self.property} does not exist"
+        return f"The source view referred to by '{self.view_id.external_id}.{self.property}' does not exist."
 
     def dump(self) -> dict[str, Any]:
         output = super().dump()

cognite/neat/rules/models/entities.py

@@ -2,7 +2,7 @@
 import sys
 from abc import ABC, abstractmethod
 from functools import total_ordering
-from typing import Annotated, Any, ClassVar, Generic, Literal, TypeVar
+from typing import Annotated, Any, ClassVar, Generic, TypeVar, cast
 
 from cognite.client.data_classes.data_modeling.ids import ContainerId, DataModelId, PropertyId, ViewId
 from pydantic import AnyHttpUrl, BaseModel, BeforeValidator, Field, PlainSerializer, model_serializer, model_validator
@@ -78,20 +78,22 @@ class Entity(BaseModel, extra="ignore"):
 
     type_: ClassVar[EntityTypes] = EntityTypes.undefined
     prefix: str | _UndefinedType = Undefined
-    suffix: str | _UnknownType
+    suffix: str
 
     @classmethod
-    def load(cls, data: Any, **defaults) -> Self:
+    def load(cls: "type[T_Entity]", data: Any, **defaults) -> "T_Entity | UnknownEntity":
         if isinstance(data, cls):
             return data
+        elif isinstance(data, str) and data == str(Unknown):
+            return UnknownEntity(prefix=Undefined, suffix=Unknown)
         if defaults and isinstance(defaults, dict):
             # This is trick to pass in default values
             return cls.model_validate({_PARSE: data, "defaults": defaults})
         else:
             return cls.model_validate(data)
 
     @model_validator(mode="before")
-    def _load(cls, data: Any) -> dict:
+    def _load(cls, data: Any) -> "dict | Entity":
         defaults = {}
         if isinstance(data, dict) and _PARSE in data:
             defaults = data.get("defaults", {})
@@ -104,6 +106,11 @@ def _load(cls, data: Any) -> dict:
             data = data.versioned_id
         elif not isinstance(data, str):
             raise ValueError(f"Cannot load {cls.__name__} from {data}")
+        elif data == str(Unknown) and cls.type_ == EntityTypes.undefined:
+            return dict(prefix=Undefined, suffix=Unknown)  # type: ignore[arg-type]
+        elif data == str(Unknown):
+            raise ValueError(f"Unknown is not allowed for {cls.type_} entity")
+
         result = cls._parse(data)
         output = defaults.copy()
         # Populate by alias
@@ -205,6 +212,9 @@ def as_non_versioned_entity(self) -> str:
         return f"{self.prefix}:{self.suffix!s}"
 
 
+T_Entity = TypeVar("T_Entity", bound=Entity)
+
+
 class ClassEntity(Entity):
     type_: ClassVar[EntityTypes] = EntityTypes.class_
     version: str | None = None
@@ -229,6 +239,16 @@ def as_class_entity(self) -> ClassEntity:
         return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
 
 
+class UnknownEntity(ClassEntity):
+    type_: ClassVar[EntityTypes] = EntityTypes.undefined
+    prefix: _UndefinedType = Undefined
+    suffix: _UnknownType = Unknown  # type: ignore[assignment]
+
+    @property
+    def id(self) -> str:
+        return str(Unknown)
+
+
 T_ID = TypeVar("T_ID", bound=ContainerId | ViewId | DataModelId | PropertyId | None)
 
 
@@ -237,6 +257,12 @@ class DMSEntity(Entity, Generic[T_ID], ABC):
     prefix: str = Field(alias="space")
     suffix: str = Field(alias="externalId")
 
+    @classmethod
+    def load(cls: "type[T_DMSEntity]", data: Any, **defaults) -> "T_DMSEntity | DMSUnknownEntity":  # type: ignore[override]
+        if isinstance(data, str) and data == str(Unknown):
+            return DMSUnknownEntity.from_id(None)
+        return cast(T_DMSEntity, super().load(data, **defaults))
+
     @property
     def space(self) -> str:
         """Returns entity space in CDF."""
@@ -260,6 +286,9 @@ def as_class(self) -> ClassEntity:
         return ClassEntity(prefix=self.space, suffix=self.external_id)
 
 
+T_DMSEntity = TypeVar("T_DMSEntity", bound=DMSEntity)
+
+
 class ContainerEntity(DMSEntity[ContainerId]):
     type_: ClassVar[EntityTypes] = EntityTypes.container
 
@@ -293,20 +322,24 @@ def from_id(cls, id: ViewId) -> "ViewEntity":
         return cls(space=id.space, externalId=id.external_id, version=id.version)
 
 
-# This is needed to handle direct relations with source=None
 class DMSUnknownEntity(DMSEntity[None]):
+    """This is a special entity that represents an unknown entity.
+
+    The use case is for direct relations where the source is not known."""
+
     type_: ClassVar[EntityTypes] = EntityTypes.undefined
-    prefix: Literal[""] = Field("", alias="space")
-    suffix: Literal[""] = Field("", alias="externalId")
+    prefix: _UndefinedType = Field(Undefined, alias="space")  # type: ignore[assignment]
+    suffix: _UnknownType = Field(Unknown, alias="externalId")  # type: ignore[assignment]
 
     def as_id(self) -> None:
         return None
 
     @classmethod
     def from_id(cls, id: None) -> "DMSUnknownEntity":
-        return cls(space="", externalId="")
+        return cls(space=Undefined, externalId=Unknown)
 
-    def __str__(self) -> str:
+    @property
+    def id(self) -> str:
         return str(Unknown)
 
 

cognite/neat/rules/models/rules/_dms_architect_rules.py

@@ -25,8 +25,7 @@
     DMSUnknownEntity,
     ParentClassEntity,
     ReferenceEntity,
-    Undefined,
-    Unknown,
+    UnknownEntity,
     URLEntity,
     ViewEntity,
     ViewEntityList,
@@ -1042,10 +1041,7 @@ def as_information_architect_rules(
                     suffix=property_.value_type.suffix,
                 )
             elif isinstance(property_.value_type, DMSUnknownEntity):
-                value_type = ClassEntity(
-                    prefix=Undefined,
-                    suffix=Unknown,
-                )
+                value_type = UnknownEntity()
             else:
                 raise ValueError(f"Unsupported value type: {property_.value_type.type_}")
 

cognite/neat/rules/models/rules/_dms_rules_write.py

@@ -9,6 +9,8 @@
 from cognite.neat.rules.models.entities import (
     ClassEntity,
     ContainerEntity,
+    DMSUnknownEntity,
+    Unknown,
     ViewEntity,
     ViewPropertyEntity,
 )
@@ -131,9 +133,11 @@ def load(
         )
 
     def dump(self, default_space: str, default_version: str) -> dict[str, Any]:
-        value_type: DataType | ViewPropertyEntity | ViewEntity
+        value_type: DataType | ViewPropertyEntity | ViewEntity | DMSUnknownEntity
         if DataType.is_data_type(self.value_type):
             value_type = DataType.load(self.value_type)
+        elif self.value_type == str(Unknown):
+            value_type = DMSUnknownEntity()
         else:
             try:
                 value_type = ViewPropertyEntity.load(self.value_type, space=default_space, version=default_version)

cognite/neat/rules/models/rules/_information_rules.py

@@ -17,17 +17,20 @@
 from cognite.neat.rules.models.entities import (
     ClassEntity,
     ContainerEntity,
+    DMSUnknownEntity,
     Entity,
     EntityTypes,
     ParentClassEntity,
     ParentEntityList,
     ReferenceEntity,
     Undefined,
     Unknown,
+    UnknownEntity,
     URLEntity,
     ViewEntity,
     ViewPropertyEntity,
     _UndefinedType,
+    _UnknownType,
 )
 from cognite.neat.rules.models.rdfpath import (
     AllReferences,
@@ -148,7 +151,7 @@ class InformationProperty(SheetEntity):
     """
 
     property_: PropertyType = Field(alias="Property")
-    value_type: DataType | ClassEntity = Field(alias="Value Type")
+    value_type: DataType | ClassEntity | UnknownEntity = Field(alias="Value Type", union_mode="left_to_right")
     min_count: int | None = Field(alias="Min Count", default=None)
     max_count: int | float | None = Field(alias="Max Count", default=None)
     default: Any | None = Field(alias="Default", default=None)
@@ -281,15 +284,15 @@ def validate_schema_completeness(self) -> Self:
         # update expected_value_types
 
         if self.metadata.schema_ == SchemaCompleteness.complete:
-            defined_classes = {class_.class_.versioned_id for class_ in self.classes}
-            referred_classes = {property_.class_.versioned_id for property_ in self.properties} | {
-                parent.versioned_id for class_ in self.classes for parent in class_.parent or []
+            defined_classes = {str(class_.class_) for class_ in self.classes}
+            referred_classes = {str(property_.class_) for property_ in self.properties} | {
+                str(parent) for class_ in self.classes for parent in class_.parent or []
             }
             referred_types = {
                 str(property_.value_type)
                 for property_ in self.properties
-                if property_.type_ == EntityTypes.object_property
-                and not (isinstance(property_.value_type, Entity) and property_.value_type.suffix is Unknown)
+                if isinstance(property_.value_type, Entity)
+                and not isinstance(property_.value_type.suffix, _UnknownType)
             }
             if not referred_classes.issubset(defined_classes) or not referred_types.issubset(defined_classes):
                 missing_classes = referred_classes.difference(defined_classes).union(
@@ -302,7 +305,7 @@ def validate_schema_completeness(self) -> Self:
     @model_validator(mode="after")
     def validate_class_has_properties_or_parent(self) -> Self:
         defined_classes = {class_.class_ for class_ in self.classes if class_.reference is None}
-        referred_classes = {property_.class_ for property_ in self.properties}
+        referred_classes = {property_.class_ for property_ in self.properties if property_.class_.suffix is not Unknown}
         has_parent_classes = {class_.class_ for class_ in self.classes if class_.parent}
         missing_classes = defined_classes.difference(referred_classes) - has_parent_classes
         if missing_classes:
@@ -467,9 +470,11 @@ def _as_dms_property(cls, prop: InformationProperty, default_space: str, default
         from ._dms_architect_rules import DMSProperty
 
         # returns property type, which can be ObjectProperty or DatatypeProperty
-        value_type: DataType | ViewEntity
+        value_type: DataType | ViewEntity | ViewPropertyEntity | DMSUnknownEntity
         if isinstance(prop.value_type, DataType):
             value_type = prop.value_type
+        elif isinstance(prop.value_type, UnknownEntity):
+            value_type = DMSUnknownEntity()
         elif isinstance(prop.value_type, ClassEntity):
             value_type = prop.value_type.as_view_entity(default_space, default_version)
         else:

cognite/neat/workflows/steps/lib/rules_importer.py

@@ -6,7 +6,7 @@
 
 from cognite.neat.rules import importers
 from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
-from cognite.neat.rules.models.entities import DataModelEntity, Undefined
+from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
 from cognite.neat.rules.models.rules import RoleTypes
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
@@ -208,7 +208,7 @@ def run(self, cdf_client: CogniteClient) -> (FlowMessage, MultiRuleData):  # typ
             return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
 
         datamodel_entity = DataModelEntity.load(datamodel_id_str)
-        if datamodel_entity.space is Undefined:
+        if isinstance(datamodel_entity, DMSUnknownEntity):
             error_text = (
                 f"Data model id should be in the format 'my_space:my_data_model(version=1)' "
                 f"or 'my_space:my_data_model', failed to parse space from {datamodel_id_str}"

docs/CHANGELOG.md

@@ -15,13 +15,16 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [0.75.7] - 29-05-24
+## [0.75.8] - 02-05-24
+### Fixed
+- `DMSExporter` now correctly exports direct relations with unknown source.
+
+## [0.75.7] - 29-04-24
 ### Added
 - `DMSExporter` now supports deletion of data model and data model components
 - `DeleteDataModelFromCDF` added to the step library
 
-
-## [0.75.6] - 26-05-24
+## [0.75.6] - 26-04-24
 ### Changed
 - All `NEAT` importers does not have `is_reference` parameter in `.to_rules()` method. This has been moved
   to the `ExcelExporter` `__init__` method. This is because this is the only place where this parameter was used.
@@ -33,33 +36,33 @@ Changes are grouped as follows:
 - When importing an `Excel` rules set with a reference model, the `ExcelImporter` would produce the warning
   `The copy method is deprecated; use the model_copy instead`. This is now fixed.
 
-## [0.75.5] - 24-05-24
+## [0.75.5] - 24-04-24
 ### Fixed
 - Potential of having duplicated spaces are now fixed
 
-## [0.75.4] - 24-05-24
+## [0.75.4] - 24-04-24
 ### Fixed
 - Rendering of correct metadata in UI for information architect
 ### Added
 - Added `OntologyToRules` that works with V2 Rules (profiling)
 
-## [0.75.3] - 23-05-24
+## [0.75.3] - 23-04-24
 ### Fixed
 - Names and descriptions were not considered for views and view properties
 
-## [0.75.2] - 23-05-24
+## [0.75.2] - 23-04-24
 ### Fixed
 - Allowing that multiple View properties can map to the same Container property
 
-## [0.75.1] - 23-05-24
+## [0.75.1] - 23-04-24
 ### Fixed
 - No spaces in any of the subfolders of the `neat` package.
 
-## [0.75.0] - 23-05-24
+## [0.75.0] - 23-04-24
 ### Added
 - Added and moved all v1 rules related code base under `legacy` module
 
-## [0.74.0] - 23-05-24
+## [0.74.0] - 23-04-24
 ### Added
 - added UI+api support for RulesV2. Read-only in the release , editable in the next release.
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.75.7"
+version = "0.75.8"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [

tests/data/__init__.py

@@ -22,3 +22,6 @@
     yaml.safe_load((DATA_DIR / "power-CapacityBid-containers.yaml").read_text())
 )
 CAPACITY_BID_JSON = DATA_DIR / "mock_capacity_bid.json"
+
+INFORMATION_UNKNOWN_VALUE_TYPE = DATA_DIR / "information-unknown-value-type.xlsx"
+DMS_UNKNOWN_VALUE_TYPE = DATA_DIR / "dms-unknown-value-type.xlsx"