Add knowledge_graph attr to Graph

.github/workflows/ci.yaml

@@ -54,6 +54,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest pytest-cov
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Install system dependencies
+        run: sudo apt update && sudo apt install --no-install-recommends graphviz
       - name: Install Python dependencies
         run: |
           pip install -e .[pre-commit,tests]

docs/gallery/autogen/annotate_inputs_outputs.py

@@ -117,7 +117,6 @@ def AddMultiplyInputs(x: int, y: int):
 
 wg.engine.recorder
 
-
 # %%
 # We can see that even though we passed the inputs as a single dictionary,
 # they were serialized as two separate ``Int`` nodes, ``x`` and ``y``, before being passed to the task.

docs/gallery/autogen/annotate_semantics.py

@@ -40,24 +40,26 @@
 #
 # - To extend the registry globally, call ``register_namespace("mat",
 #   "https://example.org/mat#")`` once at import time.
-# - To add a custom prefix for a single annotation, include it directly in the
-#   ``context`` dictionary of ``SemanticTag`` or the raw semantics payload.
-# - Explicit values always win over the defaults, so you can override or
-#   refine the URL per annotation if needed.
+# - To add a custom prefix, include it directly in the ``context`` dictionary of
+#   ``SemanticTag`` or the raw semantics payload.
+# - Explicit values always win over the defaults, so you can override or refine
+#   the URL per annotation if needed.
 
 # %%
 # How node-graph stores semantics
 # ------------------------------
 # 1. **Authoring**: sockets declare ``meta(semantics={...})`` (label, iri,
-#    rdf_types, context, attributes, relations). The graph keeps these in
-#    ``Graph.semantics_buffer`` so they survive serialization.
+#    rdf_types, context, attributes, relations). The graph keeps these in a
+#    dedicated ``Graph.knowledge_graph`` (runtime semantics survive serialization).
 # 2. **Execution**: engines match sockets to produced/consumed ``Data`` nodes,
 #    build JSON-LD snippets, and store them under ``node.base.extras['semantics']``.
 # 3. **Cross-socket references**: values inside ``relations`` or ``attributes``
 #    may refer to sockets (``"outputs.result"``).
-# 4. **Runtime attachments**: ``node_graph.semantics.attach_semantics`` lets you
-#    declare relations inside workflow code; the graph buffer remembers them
-#    until execution completes.
+# 4. **Runtime semantics**: ``node_graph.semantics.attach_semantics`` lets you
+#    declare relations inside workflow code; the knowledge-graph keeps them until
+#    execution completes.
+# 5. **RDF + visualisation**: the same ``KnowledgeGraph`` can be exported to rdflib
+#    or rendered with graphviz for quick inspection.
 
 # %%
 # Declaring socket semantics
@@ -69,10 +71,10 @@
 from typing import Annotated, Any
 from node_graph import task
 from node_graph.socket_spec import meta, namespace
+from ase import Atoms
 
 energy_semantics_meta = meta(
     semantics={
-        "label": "Potential energy",
         "iri": "qudt:PotentialEnergy",
         "rdf_types": ["qudt:QuantityValue"],
         "context": {
@@ -87,10 +89,18 @@
 
 
 @task()
-def AnnotateSemantics(energy: float) -> Annotated[float, energy_semantics_meta]:
+def calc_energy(atoms: Atoms) -> Annotated[float, energy_semantics_meta]:
+    from ase.calculators.emt import EMT
+
+    energy = atoms.get_potential_energy(calculator=EMT())
     return energy
 
 
+# ``attributes`` is the right place for literal predicate/value pairs (units,
+# DOIs, numbers). Use ``relations`` when the value is a reference to another
+# resource or socket (e.g., CURIE/IRI pointing to a dataset or another port).
+
+
 # %%
 # Typed semantics with Pydantic
 # -----------------------------
@@ -129,9 +139,17 @@ def TypedSemantics(energy: float) -> Annotated[float, meta(semantics=EnergyEV)]:
 # ``structure`` has a relation ``mat:hasProperty`` pointing to the output of the
 # ``compute_band_structure`` task.
 
+BAND_META = meta(
+    semantics={
+        "label": "Band Structure",
+        "description": "The electronic band structure of a material.",
+        "iri": "emmo:BandStructure",
+    }
+)
+
 
 @task()
-def compute_band_structure(structure):
+def compute_band_structure(structure) -> Annotated[float, BAND_META]:
     return 1.0
 
 
@@ -167,13 +185,22 @@ def workflow(structure: Annotated[Any, structure_meta]):
 from node_graph.semantics import attach_semantics
 
 
+DENSITY_OF_STATES_META = meta(
+    semantics={
+        "label": "Density of States",
+        "description": "The electronic density of states of a material.",
+        "iri": "emmo:DensityOfStates",
+    }
+)
+
+
 @task()
 def generate(structure):
     return structure
 
 
 @task()
-def compute_density_of_states(structure):
+def compute_density_of_states(structure) -> Annotated[float, DENSITY_OF_STATES_META]:
     return 1.0
 
 
@@ -196,6 +223,20 @@ def workflow_with_runtime_semantics(
     return {"output_structure": mutated, "bands": bands, "dos": dos}
 
 
+# %%
+# Exporting or visualising the knowledge graph
+# --------------------------------------------
+# Every materialised graph carries a ``knowledge_graph`` attribute that owns the
+# semantics buffer and can emit an rdflib graph or a Graphviz Digraph.
+
+
+ng = workflow_with_runtime_semantics.build(structure="test")
+ng.knowledge_graph
+
+# ``rdf_graph`` can be serialized (e.g., ``rdf_graph.serialize(format=\"turtle\")``),
+# while ``viz`` renders/exports to DOT/SVG/PDF for documentation.
+
+
 # %%
 # Tips and use cases
 # ------------------

docs/requirements.txt

@@ -5,3 +5,4 @@ furo
 sphinx_gallery
 matplotlib
 graphviz>=0.20
+ase

pyproject.toml

@@ -33,6 +33,8 @@ dependencies = [
     "typing_extensions>=4.7; python_version < '3.11'",
     "pydantic~=2.6",
     "pyyaml",
+    "rdflib>=6.0",
+    "graphviz>=0.20",
 ]
 
 [project.urls]

src/node_graph/__init__.py

@@ -1,6 +1,7 @@
 from .graph import Graph
 from .task import Task
 from .decorator import task
+from .knowledge_graph import KnowledgeGraph
 from .executor import SafeExecutor, RuntimeExecutor
 from .tasks import TaskPool
 from .collection import group
@@ -19,4 +20,5 @@
     "group",
     "namespace",
     "dynamic",
+    "KnowledgeGraph",
 ]

src/node_graph/decorator.py

@@ -5,6 +5,7 @@
 from .task import Task
 from .task_spec import TaskHandle, BaseHandle
 from .socket_spec import SocketSpec
+from .utils.function import inspect_callable_metadata
 
 
 def build_task_from_callable(
@@ -53,13 +54,18 @@ def decorator_task(
     def wrap(func) -> TaskHandle:
         from node_graph.tasks.function_task import FunctionTask
 
+        callable_meta = inspect_callable_metadata(func)
+        metadata = {"callable": callable_meta}
+        version = callable_meta.get("package_version")
         return FunctionTask.build(
             obj=func,
             identifier=identifier or func.__name__,
             catalog=catalog,
             input_spec=inputs,
             output_spec=outputs,
             error_handlers=error_handlers,
+            metadata=metadata,
+            version=version,
         )
 
     return wrap
@@ -83,13 +89,18 @@ def decorator_graph(
     def wrap(func) -> TaskHandle:
         from node_graph.tasks.function_task import FunctionTask
 
+        callable_meta = inspect_callable_metadata(func)
+        metadata = {"callable": callable_meta}
+        version = callable_meta.get("package_version")
         return FunctionTask.build(
             obj=func,
             identifier=identifier or func.__name__,
             task_type="graph",
             catalog=catalog,
             input_spec=inputs,
             output_spec=outputs,
+            metadata=metadata,
+            version=version,
         )
 
     return wrap

src/node_graph/graph.py

@@ -12,7 +12,7 @@
 from node_graph.utils import yaml_to_dict
 from .config import BuiltinPolicy, BUILTIN_TASKS, MAX_LINK_LIMIT
 from .mixins import IOOwnerMixin, WidgetRenderableMixin
-from node_graph.semantics import serialize_semantics_buffer
+from node_graph.knowledge_graph import KnowledgeGraph
 from dataclasses import dataclass
 from dataclasses import replace
 
@@ -101,6 +101,7 @@ def __init__(
         parent: Optional[Task] = None,
         interactive_widget: bool = False,
         init_graph_level_tasks: bool = True,
+        metadata: Optional[Dict[str, Any]] = None,
     ) -> None:
         """Initializes a new instance of the Graph class.
 
@@ -125,7 +126,8 @@ def __init__(
         self._init_graph_spec(inputs, outputs, ctx)
         if init_graph_level_tasks:
             self._init_graph_level_tasks()
-        self.semantics_buffer = {"relations": [], "payloads": []}
+        self.knowledge_graph = KnowledgeGraph(graph_uuid=self.uuid, graph=self)
+        self._metadata: Dict[str, Any] = dict(metadata or {})
 
         self.state = "CREATED"
         self.action = "NONE"
@@ -397,6 +399,8 @@ def to_dict(
         )
 
         links = self.links_to_dict()
+        kg_payload = self.knowledge_graph.to_dict()
+
         data = {
             "platform_version": f"{self.platform}@{self.platform_version}",
             "uuid": self.uuid,
@@ -409,8 +413,8 @@ def to_dict(
             "tasks": tasks,
             "links": links,
             "description": self.description,
+            "knowledge_graph": kg_payload,
         }
-        data["semantics_buffer"] = serialize_semantics_buffer(self.semantics_buffer)
         return data
 
     def get_metadata(self) -> Dict[str, Any]:
@@ -423,6 +427,10 @@ def get_metadata(self) -> Dict[str, Any]:
             "callable_name": self.__class__.__name__,
             "module_path": self.__class__.__module__,
         }
+        for key, value in (self._metadata or {}).items():
+            if key in {"graph_type", "graph_class"}:
+                continue
+            meta[key] = value
         return meta
 
     def export_tasks_to_dict(
@@ -498,13 +506,17 @@ def from_dict(cls, ngdata: Dict[str, Any]) -> Graph:
             Graph: The rebuilt task graph.
         """
         spec = GraphSpec.from_dict(ngdata.get("spec", {}))
+        raw_meta = ngdata.get("metadata", {}) or {}
+        base_meta = {k: raw_meta[k] for k in ("graph_type",) if k in raw_meta}
+        extra_meta = {k: v for k, v in raw_meta.items() if k not in {"graph_type"}}
         ng = cls(
             name=ngdata["name"],
             uuid=ngdata.get("uuid"),
             inputs=spec.inputs,
             outputs=spec.outputs,
             ctx=spec.ctx,
-            graph_type=ngdata["metadata"].get("graph_type", "NORMAL"),
+            graph_type=base_meta.get("graph_type", "NORMAL"),
+            metadata=extra_meta,
         )
         ng.state = ngdata.get("state", "CREATED")
         ng.action = ngdata.get("action", "NONE")
@@ -514,9 +526,14 @@ def from_dict(cls, ngdata: Dict[str, Any]) -> Graph:
             ng.add_task_from_dict(ndata)
 
         ng.links_from_dict(ngdata.get("links", []))
-        semantics_buffer = ngdata.get("semantics_buffer")
-        if semantics_buffer is not None:
-            ng.semantics_buffer = semantics_buffer
+        kg_payload = ngdata.get("knowledge_graph")
+        if kg_payload is not None:
+            ng.knowledge_graph = KnowledgeGraph.from_dict(
+                kg_payload, graph_uuid=ng.uuid
+            )
+            ng.knowledge_graph._graph = ng
+            ng.knowledge_graph.graph_uuid = ng.uuid
+            ng.knowledge_graph._dirty = True
         return ng
 
     def add_task_from_dict(self, ndata: Dict[str, Any]) -> Task:
@@ -582,6 +599,8 @@ def copy(self, name: Optional[str] = None) -> "Graph":
                 ng.tasks[link.from_task.name].outputs[link.from_socket._scoped_name],
                 ng.tasks[link.to_task.name].inputs[link.to_socket._scoped_name],
             )
+        ng.knowledge_graph = self.knowledge_graph.copy(graph_uuid=ng.uuid)
+        ng.knowledge_graph._graph = ng
         return ng
 
     @classmethod