Neat 206 delete dms data model step is missing in v2 rules (#421)

[0.75.7] - 29-05-24
Added
DMSExporter now supports deletion of data model and data model components
DeleteDataModelFromCDF added to the step library

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.75.6"
+version="0.75.7"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.75.6"
+__version__ = "0.75.7"

cognite/neat/rules/exporters/_models.py

@@ -30,6 +30,7 @@ class UploadResult(UploadResultCore):
     skipped: int = 0
     failed_created: int = 0
     failed_changed: int = 0
+    failed_deleted: int = 0
     error_messages: list[str] = field(default_factory=list)
 
     @property
@@ -38,7 +39,7 @@ def total(self) -> int:
 
     @property
     def failed(self) -> int:
-        return self.failed_created + self.failed_changed
+        return self.failed_created + self.failed_changed + self.failed_deleted
 
     def as_report_str(self) -> str:
         line = []
@@ -50,9 +51,13 @@ def as_report_str(self) -> str:
             line.append(f"skipped {self.skipped}")
         if self.unchanged:
             line.append(f"unchanged {self.unchanged}")
+        if self.deleted:
+            line.append(f"deleted {self.deleted}")
         if self.failed_created:
             line.append(f"failed to create {self.failed_created}")
         if self.failed_changed:
             line.append(f"failed to update {self.failed_changed}")
+        if self.failed_deleted:
+            line.append(f"failed to delete {self.failed_deleted}")
 
         return f"{self.name.title()}: {', '.join(line)}"

cognite/neat/rules/exporters/_rules2dms.py

@@ -151,21 +151,55 @@ def export(self, rules: Rules) -> DMSSchema:
             schema.frozen_ids.update(set(reference_schema.node_types.as_ids()))
         return schema
 
+    def delete_from_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
+
+        # we need to reverse order in which we are picking up the items to delete
+        # as they are sorted in the order of creation and we need to delete them in reverse order
+        for all_items, loader in reversed(to_export):
+            all_item_ids = loader.get_ids(all_items)
+            skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
+            item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
+            cdf_items = loader.retrieve(item_ids)
+            cdf_item_by_id = {loader.get_id(item): item for item in cdf_items}
+            items = [item for item in all_items if loader.get_id(item) in item_ids]
+            to_delete = []
+
+            for item in items:
+                if (
+                    isinstance(loader, DataModelingLoader)
+                    and self.include_space is not None
+                    and not loader.in_space(item, self.include_space)
+                ):
+                    continue
+
+                cdf_item = cdf_item_by_id.get(loader.get_id(item))
+                if cdf_item:
+                    to_delete.append(cdf_item)
+
+            deleted = len(to_delete)
+            failed_deleted = 0
+
+            error_messages: list[str] = []
+            if not dry_run:
+                if to_delete:
+                    try:
+                        loader.delete(to_delete)
+                    except CogniteAPIError as e:
+                        failed_deleted = len(e.failed) + len(e.unknown)
+                        deleted -= failed_deleted
+                        error_messages.append(f"Failed delete: {e.message}")
+
+            yield UploadResult(
+                name=loader.resource_name,
+                deleted=deleted,
+                skipped=skipped,
+                failed_deleted=failed_deleted,
+                error_messages=error_messages,
+            )
+
     def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = False) -> Iterable[UploadResult]:
-        schema = self.export(rules)
-        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
-        if self.export_components.intersection({"all", "spaces"}):
-            to_export.append((schema.spaces, SpaceLoader(client)))
-        if self.export_components.intersection({"all", "containers"}):
-            to_export.append((schema.containers, ContainerLoader(client)))
-        if self.export_components.intersection({"all", "views"}):
-            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
-        if self.export_components.intersection({"all", "data_models"}):
-            to_export.append((schema.data_models, DataModelLoader(client)))
-        if isinstance(schema, PipelineSchema):
-            to_export.append((schema.databases, RawDatabaseLoader(client)))
-            to_export.append((schema.raw_tables, RawTableLoader(client)))
-            to_export.append((schema.transformations, TransformationLoader(client)))
+        schema, to_export = self._prepare_schema_and_exporters(rules, client)
 
         # The conversion from DMS to GraphQL does not seem to be triggered even if the views
         # are changed. This is a workaround to force the conversion.
@@ -254,3 +288,22 @@ def export_to_cdf(self, rules: Rules, client: CogniteClient, dry_run: bool = Fal
 
             if loader.resource_name == "views" and (created or changed) and not redeploy_data_model:
                 redeploy_data_model = True
+
+    def _prepare_schema_and_exporters(
+        self, rules, client
+    ) -> tuple[DMSSchema, list[tuple[CogniteResourceList, ResourceLoader]]]:
+        schema = self.export(rules)
+        to_export: list[tuple[CogniteResourceList, ResourceLoader]] = []
+        if self.export_components.intersection({"all", "spaces"}):
+            to_export.append((schema.spaces, SpaceLoader(client)))
+        if self.export_components.intersection({"all", "containers"}):
+            to_export.append((schema.containers, ContainerLoader(client)))
+        if self.export_components.intersection({"all", "views"}):
+            to_export.append((schema.views, ViewLoader(client, self.existing_handling)))
+        if self.export_components.intersection({"all", "data_models"}):
+            to_export.append((schema.data_models, DataModelLoader(client)))
+        if isinstance(schema, PipelineSchema):
+            to_export.append((schema.databases, RawDatabaseLoader(client)))
+            to_export.append((schema.raw_tables, RawTableLoader(client)))
+            to_export.append((schema.transformations, TransformationLoader(client)))
+        return schema, to_export

cognite/neat/utils/cdf_loaders/_data_modeling.py

@@ -87,8 +87,10 @@ def retrieve(self, ids: SequenceNotStr[str]) -> SpaceList:
     def update(self, items: Sequence[SpaceApply]) -> SpaceList:
         return self.create(items)
 
-    def delete(self, ids: SequenceNotStr[str]) -> list[str]:
-        return self.client.data_modeling.spaces.delete(ids)
+    def delete(self, ids: SequenceNotStr[str] | Sequence[Space | SpaceApply]) -> list[str]:
+        if all(isinstance(item, Space) for item in ids) or all(isinstance(item, SpaceApply) for item in ids):
+            ids = [cast(Space | SpaceApply, item).space for item in ids]
+        return self.client.data_modeling.spaces.delete(cast(SequenceNotStr[str], ids))
 
 
 class ViewLoader(DataModelingLoader[ViewId, ViewApply, View, ViewApplyList, ViewList]):

cognite/neat/workflows/steps/lib/rules_exporter.py

@@ -17,12 +17,109 @@
     "RulesToSHACL",
     "RulesToSemanticDataModel",
     "RulesToCDFTransformations",
+    "DeleteDataModelFromCDF",
 ]
 
 
 CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
 
 
+class DeleteDataModelFromCDF(Step):
+    """
+    This step deletes data model and its components from CDF
+    """
+
+    description = "This step deletes data model and its components from CDF."
+    version = "private-beta"
+    category = CATEGORY
+
+    configurables: ClassVar[list[Configurable]] = [
+        Configurable(
+            name="Dry run",
+            value="False",
+            label=("Whether to perform a dry run of the deleter. "),
+            options=["True", "False"],
+        ),
+        Configurable(
+            name="Components",
+            type="multi_select",
+            value="",
+            label="Select which DMS schema component(s) to be deleted from CDF",
+            options=["spaces", "containers", "views", "data_models"],
+        ),
+        Configurable(
+            name="Multi-space components deletion",
+            value="False",
+            label=(
+                "Whether to delete only components belonging to the data model space"
+                " (i.e. space define under Metadata sheet of Rules), "
+                "or also additionally delete components outside of the data model space."
+            ),
+            options=["True", "False"],
+        ),
+    ]
+
+    def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage:  # type: ignore[override]
+        if self.configs is None or self.data_store_path is None:
+            raise StepNotInitialized(type(self).__name__)
+        components_to_delete = {
+            cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
+            for key, value in self.complex_configs["Components"].items()
+            if value
+        }
+        dry_run = self.configs["Dry run"] == "True"
+        multi_space_components_delete: bool = self.configs["Multi-space components deletion"] == "True"
+
+        if not components_to_delete:
+            return FlowMessage(
+                error_text="No DMS Schema components selected for removal! Please select minimum one!",
+                step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
+            )
+        input_rules = rules.dms or rules.information
+        if input_rules is None:
+            return FlowMessage(
+                error_text="Missing DMS or Information rules in the input data! "
+                "Please ensure that a DMS or Information rules is provided!",
+                step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
+            )
+
+        dms_exporter = exporters.DMSExporter(
+            export_components=frozenset(components_to_delete),
+            include_space=(
+                None
+                if multi_space_components_delete
+                else {input_rules.metadata.space if isinstance(input_rules, DMSRules) else input_rules.metadata.prefix}
+            ),
+        )
+
+        report_lines = ["# Data Model Deletion from CDF\n\n"]
+        errors = []
+        for result in dms_exporter.delete_from_cdf(rules=input_rules, client=cdf_client, dry_run=dry_run):
+            report_lines.append(result.as_report_str())
+            errors.extend(result.error_messages)
+
+        report_lines.append("\n\n# ERRORS\n\n")
+        report_lines.extend(errors)
+
+        output_dir = self.data_store_path / Path("staging")
+        output_dir.mkdir(parents=True, exist_ok=True)
+        report_file = "dms_component_creation_report.txt"
+        report_full_path = output_dir / report_file
+        report_full_path.write_text("\n".join(report_lines))
+
+        output_text = (
+            "<p></p>"
+            "Download Data Model Deletion "
+            f'<a href="/data/staging/{report_file}?{time.time()}" '
+            f'target="_blank">Report</a>'
+        )
+
+        if errors:
+            return FlowMessage(error_text=output_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
+        else:
+            return FlowMessage(output_text=output_text)
+
+
 class RulesToDMS(Step):
     """
     This step exports Rules to DMS Schema components in CDF

docs/CHANGELOG.md

@@ -15,6 +15,12 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.75.7] - 29-05-24
+### Added
+- `DMSExporter` now supports deletion of data model and data model components
+- `DeleteDataModelFromCDF` added to the step library
+
+
 ## [0.75.6] - 26-05-24
 ### Changed
 - All `NEAT` importers does not have `is_reference` parameter in `.to_rules()` method. This has been moved

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-neat"
-version = "0.75.6"
+version = "0.75.7"
 readme = "README.md"
 description = "Knowledge graph transformation"
 authors = [