Neat 141 legacy rules UI switch (#426)

[0.75.9] - 04-05-24 Improved Steps are now categorized as current, legacy, and io steps Workflow fails if one mix current and legacy steps in the same workflow
cognitedata · May 3, 2024 · d30432e · d30432e
1 parent 272b420
commit d30432e
Show file tree

Hide file tree

Showing 34 changed files with 111 additions and 75 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
 
-version="0.75.8"
+version="0.75.9"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

diff --git a/cognite/neat/_version.py b/cognite/neat/_version.py
@@ -1 +1 @@
-__version__ = "0.75.8"
+__version__ = "0.75.9"
diff --git a/cognite/neat/app/api/routers/rules.py b/cognite/neat/app/api/routers/rules.py
@@ -14,9 +14,9 @@
 from cognite.neat.rules import importers
 from cognite.neat.rules.models.rules import RoleTypes
 from cognite.neat.workflows.steps.data_contracts import RulesData
-from cognite.neat.workflows.steps.lib.rules_exporter import RulesToExcel
-from cognite.neat.workflows.steps.lib.rules_importer import ExcelToRules
-from cognite.neat.workflows.steps.lib.v1.rules_importer import ImportExcelToRules
+from cognite.neat.workflows.steps.lib.current.rules_exporter import RulesToExcel
+from cognite.neat.workflows.steps.lib.current.rules_importer import ExcelToRules
+from cognite.neat.workflows.steps.lib.legacy.rules_importer import ImportExcelToRules
 from cognite.neat.workflows.utils import get_file_hash
 
 router = APIRouter()

diff --git a/cognite/neat/app/ui/neat-app/build/asset-manifest.json b/cognite/neat/app/ui/neat-app/build/asset-manifest.json
@@ -1,14 +1,14 @@
 {
   "files": {
     "main.css": "./static/css/main.38a62222.css",
-    "main.js": "./static/js/main.4345d42f.js",
+    "main.js": "./static/js/main.ec7f72e2.js",
     "static/media/logo.svg": "./static/media/logo.8093b84df9ed36a174c629d6fe0b730d.svg",
     "index.html": "./index.html",
     "main.38a62222.css.map": "./static/css/main.38a62222.css.map",
-    "main.4345d42f.js.map": "./static/js/main.4345d42f.js.map"
+    "main.ec7f72e2.js.map": "./static/js/main.ec7f72e2.js.map"
   },
   "entrypoints": [
     "static/css/main.38a62222.css",
-    "static/js/main.4345d42f.js"
+    "static/js/main.ec7f72e2.js"
   ]
 }
diff --git a/cognite/neat/app/ui/neat-app/build/index.html b/cognite/neat/app/ui/neat-app/build/index.html
@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./logo192.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="NEAT controll panel"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>NEAT App</title><script defer="defer" src="./static/js/main.4345d42f.js"></script><link href="./static/css/main.38a62222.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./logo192.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="NEAT controll panel"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>NEAT App</title><script defer="defer" src="./static/js/main.ec7f72e2.js"></script><link href="./static/css/main.38a62222.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
diff --git a/...neat-app/build/static/js/main.4345d42f.js → ...neat-app/build/static/js/main.ec7f72e2.js b/...neat-app/build/static/js/main.4345d42f.js → ...neat-app/build/static/js/main.ec7f72e2.js
diff --git a/...ld/static/js/main.4345d42f.js.LICENSE.txt → ...ld/static/js/main.ec7f72e2.js.LICENSE.txt b/...ld/static/js/main.4345d42f.js.LICENSE.txt → ...ld/static/js/main.ec7f72e2.js.LICENSE.txt
diff --git a/...-app/build/static/js/main.4345d42f.js.map → ...-app/build/static/js/main.ec7f72e2.js.map b/...-app/build/static/js/main.4345d42f.js.map → ...-app/build/static/js/main.ec7f72e2.js.map
diff --git a/cognite/neat/app/ui/neat-app/src/components/StepEditorDialog.tsx b/cognite/neat/app/ui/neat-app/src/components/StepEditorDialog.tsx
@@ -334,9 +334,9 @@ return (
               <Box>
               <Stack direction="row" spacing={2}>
                 <Typography sx={{marginRight:7}}>
-                Version : <span style={{ color: selectedStepTemplate?.version.toLowerCase().includes("private-alpha") ? "red" : "green", fontWeight: selectedStepTemplate?.version.toLowerCase().includes("private-beta") ? "bold" : "bold" }}>{selectedStepTemplate?.version}</span>
-                {selectedStepTemplate?.version.toLowerCase().includes("private-alpha") && (
-                          <Tooltip title="Caution: This step is in the private alpha stage and may be subject to changes or deprecation.">
+                Version : <span style={{ color: selectedStepTemplate?.version.toLowerCase().includes("legacy") ? "red" : "green", fontWeight: selectedStepTemplate?.version.toLowerCase().includes("private-beta") ? "bold" : "bold" }}>{selectedStepTemplate?.version}</span>
+                {selectedStepTemplate?.version.toLowerCase().includes("legacy") && (
+                          <Tooltip title="Caution: This step is legacy step, meaning it can be only used with other legacy steps. We are not supporting it anymore.">
                             <WarningIcon sx={{ marginLeft: 1, marginBottom: -0.5, color: "orange" }} />
                           </Tooltip>
                 )}

diff --git a/cognite/neat/workflows/manager.py b/cognite/neat/workflows/manager.py
@@ -12,7 +12,7 @@
 from cognite.neat.config import Config
 from cognite.neat.workflows import BaseWorkflow
 from cognite.neat.workflows.base import WorkflowDefinition
-from cognite.neat.workflows.model import FlowMessage, InstanceStartMethod, WorkflowState
+from cognite.neat.workflows.model import FlowMessage, InstanceStartMethod, WorkflowState, WorkflowStepDefinition
 from cognite.neat.workflows.steps_registry import StepsRegistry
 from cognite.neat.workflows.tasks import WorkflowTaskBuilder
 
@@ -195,10 +195,21 @@ def start_workflow_instance(
         self, workflow_name: str, step_id: str = "", flow_msg: FlowMessage | None = None, sync: bool | None = None
     ) -> WorkflowStartStatus:
         retrieved = self.get_workflow(workflow_name)
+
         if retrieved is None:
             return WorkflowStartStatus(
                 workflow_instance=None, is_success=False, status_text="Workflow not found in registry"
             )
+
+        if self._is_workflow_made_of_mixed_steps(retrieved.workflow_steps):
+            retrieved.state = WorkflowState.FAILED
+            return WorkflowStartStatus(
+                workflow_instance=None,
+                is_success=False,
+                status_text="Workflow consists of both legacy and current steps. "
+                "Please update the workflow to use only current steps.",
+            )
+
         workflow = retrieved
         retrieved_step = workflow.get_trigger_step(step_id)
         if retrieved_step is None:
@@ -269,3 +280,13 @@ def start_workflow_instance(
         return WorkflowStartStatus(
             workflow_instance=None, is_success=False, status_text="Unsupported workflow start method"
         )
+
+    def _is_workflow_made_of_mixed_steps(self, steps: list[WorkflowStepDefinition]):
+        legacy_steps = 0
+        current_steps = 0
+        for step in steps:
+            if step.method in self.steps_registry.categorized_steps["legacy"]:
+                legacy_steps += 1
+            if step.method in self.steps_registry.categorized_steps["current"]:
+                current_steps += 1
+        return legacy_steps > 0 and current_steps > 0
diff --git a/cognite/neat/workflows/steps/lib/__init__.py b/cognite/neat/workflows/steps/lib/__init__.py
@@ -1,7 +0,0 @@
-from .io_steps import *  # noqa
-from .rules_importer import *  # noqa
-from .rules_validator import *  # noqa
-from .rules_exporter import *  # noqa
-from .graph_store import *  # noqa
-from .graph_loader import *  # noqa
-from .graph_extractor import *  # noqa

diff --git a/cognite/neat/workflows/steps/lib/current/__init__.py b/cognite/neat/workflows/steps/lib/current/__init__.py
@@ -0,0 +1,6 @@
+from .rules_importer import *  # noqa
+from .rules_validator import *  # noqa
+from .rules_exporter import *  # noqa
+from .graph_store import *  # noqa
+from .graph_loader import *  # noqa
+from .graph_extractor import *  # noqa
diff --git a/...at/workflows/steps/lib/graph_extractor.py → ...lows/steps/lib/current/graph_extractor.py b/...at/workflows/steps/lib/graph_extractor.py → ...lows/steps/lib/current/graph_extractor.py
diff --git a/.../neat/workflows/steps/lib/graph_loader.py → ...rkflows/steps/lib/current/graph_loader.py b/.../neat/workflows/steps/lib/graph_loader.py → ...rkflows/steps/lib/current/graph_loader.py
diff --git a/...e/neat/workflows/steps/lib/graph_store.py → ...orkflows/steps/lib/current/graph_store.py b/...e/neat/workflows/steps/lib/graph_store.py → ...orkflows/steps/lib/current/graph_store.py
diff --git a/...eat/workflows/steps/lib/rules_exporter.py → ...flows/steps/lib/current/rules_exporter.py b/...eat/workflows/steps/lib/rules_exporter.py → ...flows/steps/lib/current/rules_exporter.py
diff --git a/...eat/workflows/steps/lib/rules_importer.py → ...flows/steps/lib/current/rules_importer.py b/...eat/workflows/steps/lib/rules_importer.py → ...flows/steps/lib/current/rules_importer.py
diff --git a/...at/workflows/steps/lib/rules_validator.py → ...lows/steps/lib/current/rules_validator.py b/...at/workflows/steps/lib/rules_validator.py → ...lows/steps/lib/current/rules_validator.py
diff --git a/cognite/neat/workflows/steps/lib/io/__init__.py b/cognite/neat/workflows/steps/lib/io/__init__.py
@@ -0,0 +1 @@
+from .io_steps import *  # noqa
diff --git a/cognite/neat/workflows/steps/lib/io_steps.py → ...e/neat/workflows/steps/lib/io/io_steps.py b/cognite/neat/workflows/steps/lib/io_steps.py → ...e/neat/workflows/steps/lib/io/io_steps.py
diff --git a/...e/neat/workflows/steps/lib/v1/__init__.py → ...at/workflows/steps/lib/legacy/__init__.py b/...e/neat/workflows/steps/lib/v1/__init__.py → ...at/workflows/steps/lib/legacy/__init__.py
diff --git a/...s/steps/lib/v1/graph_contextualization.py → ...eps/lib/legacy/graph_contextualization.py b/...s/steps/lib/v1/graph_contextualization.py → ...eps/lib/legacy/graph_contextualization.py
@@ -7,11 +7,11 @@
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
 __all__ = ["SimpleGraphEntityMatcher"]
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class SimpleGraphEntityMatcher(Step):
-    version = "private-alpha"
+    version = "legacy"
     description = "The step matches entities in the graph and creates links based on provided configurations"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [

diff --git a/...workflows/steps/lib/v1/graph_extractor.py → ...flows/steps/lib/legacy/graph_extractor.py b/...workflows/steps/lib/v1/graph_extractor.py → ...flows/steps/lib/legacy/graph_extractor.py
@@ -29,7 +29,7 @@
     "ExtractGraphFromDexpiFile",
 ]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class ExtractGraphFromRdfFile(Step):
@@ -38,7 +38,7 @@ class ExtractGraphFromRdfFile(Step):
     """
 
     description = "This step extract instances from a file into the source graph. The file must be in RDF format."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -91,7 +91,7 @@ class ExtractGraphFromDexpiFile(Step):
     """
 
     description = "This step converts DEXPI P&ID (XML) into Knowledge Graph"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -130,7 +130,7 @@ class ExtractGraphFromGraphCapturingSheet(Step):
     """
 
     description = "This step extracts nodes and edges from graph capturing spreadsheet and load them into graph"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -187,7 +187,7 @@ class ExtractGraphFromMockGraph(Step):
     """
 
     description = "This step generate mock graph based on the defined classes and target number of instances"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -239,7 +239,7 @@ class ExtractGraphFromRulesInstanceSheet(Step):
 
     description = "This step extracts instances from Rules object and loads them into the graph."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -276,7 +276,7 @@ class ExtractGraphFromRulesDataModel(Step):
 
     description = "This step extracts data model from rules file and loads it into source graph."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override, syntax]
         self, transformation_rules: RulesData, source_graph: SourceGraph
@@ -321,7 +321,7 @@ class ExtractGraphFromJsonFile(Step):
 
     description = "This step extracts instances from json file and loads them into a graph store"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="file_name", value="data_dump.json", label="Full path to the file containing data dump in JSON format"
@@ -504,7 +504,7 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
 
     description = "This step extracts instances from Aveva PI AF and loads them into a graph store"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="file_name",

diff --git a/...at/workflows/steps/lib/v1/graph_loader.py → ...orkflows/steps/lib/legacy/graph_loader.py b/...at/workflows/steps/lib/v1/graph_loader.py → ...orkflows/steps/lib/legacy/graph_loader.py
@@ -52,7 +52,7 @@
     "LoadGraphToRdfFile",
 ]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class LoadLabelsToCDF(Step):
@@ -62,7 +62,7 @@ class LoadLabelsToCDF(Step):
 
     description = "This step creates default NEAT labels in CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added."))
     ]
@@ -176,7 +176,7 @@ class LoadGraphToRdfFile(Step):
 
     description = "The step generates nodes and edges from the graph"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="graph_name",
@@ -227,7 +227,7 @@ class LoadNodesToCDF(Step):
 
     description = "This step uploads nodes to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(self, cdf_client: CogniteClient, nodes: Nodes) -> FlowMessage:  # type: ignore[override, syntax]
         if nodes.nodes:
@@ -244,7 +244,7 @@ class LoadEdgesToCDF(Step):
 
     description = "This step uploads edges to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(self, cdf_client: CogniteClient, edges: Edges) -> FlowMessage:  # type: ignore[override, syntax]
         if edges.edges:
@@ -263,7 +263,7 @@ class GenerateAssetsFromGraph(Step):
         "The step generates assets from the graph ,categorizes them and stores them in CategorizedAssets object"
     )
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added.")),
         Configurable(
@@ -469,7 +469,7 @@ class LoadAssetsToCDF(Step):
 
     description = "This step uploads categorized assets to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override]
         self, cdf_client: CogniteClient, categorized_assets: CategorizedAssets, flow_msg: FlowMessage
@@ -528,7 +528,7 @@ class GenerateRelationshipsFromGraph(Step):
 
     description = "This step generates relationships from the graph and saves them to CategorizedRelationships object"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added.")),
         Configurable(
@@ -597,7 +597,7 @@ class LoadRelationshipsToCDF(Step):
 
     description = "This step uploads relationships to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override, syntax]
         self, client: CogniteClient, categorized_relationships: CategorizedRelationships

diff --git a/...eat/workflows/steps/lib/v1/graph_store.py → ...workflows/steps/lib/legacy/graph_store.py b/...eat/workflows/steps/lib/v1/graph_store.py → ...workflows/steps/lib/legacy/graph_store.py
@@ -11,7 +11,7 @@
 
 __all__ = ["ResetGraphStores", "ConfigureGraphStore"]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class ConfigureDefaultGraphStores(Step):
@@ -21,7 +21,7 @@ class ConfigureDefaultGraphStores(Step):
 
     description = "This step initializes the source and solution graph stores."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="source_rdf_store.type",
@@ -154,7 +154,7 @@ class ResetGraphStores(Step):
 
     description = "This step resets graph stores to their initial state (clears all data)."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="graph_name",
@@ -186,7 +186,7 @@ class ConfigureGraphStore(Step):
     """
 
     description = "This step initializes the source and solution graph stores."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(

diff --git a/...rkflows/steps/lib/v1/graph_transformer.py → ...ows/steps/lib/legacy/graph_transformer.py b/...rkflows/steps/lib/v1/graph_transformer.py → ...ows/steps/lib/legacy/graph_transformer.py
@@ -10,7 +10,7 @@
 
 __all__ = ["TransformSourceToSolutionGraph"]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class TransformSourceToSolutionGraph(Step):
@@ -20,7 +20,7 @@ class TransformSourceToSolutionGraph(Step):
 
     description = "The step transforms source graph to solution graph"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="cdf_lookup_database",