[resotolib][feat] ModelExport: include name and description (#1820)

resotocore/resotocore/model/exportable_model.py

@@ -14,7 +14,13 @@
 from resotocore.types import Json, JsonElement
 
 
-def json_export_simple_schema(model: Model) -> List[Json]:
+def json_export_simple_schema(
+    model: Model,
+    with_properties: bool = True,
+    with_relatives: bool = True,
+    with_metadata: bool = True,
+    aggregate_roots_only: bool = False,
+) -> List[Json]:
     def export_simple(kind: SimpleKind) -> Json:
         result = kind.as_json()
         result["type"] = "simple"
@@ -45,17 +51,21 @@ def prop_kind(kd: Kind) -> JsonElement:
         return p
 
     def export_complex(kind: ComplexKind) -> Json:
-        return dict(
+        result = dict(
             type="object",
             fqn=kind.fqn,
-            bases=kind.bases,
-            allow_unknown_props=kind.allow_unknown_props,
-            predecessor_kinds=kind.predecessor_kinds(),
-            successor_kinds=kind.successor_kinds,
             aggregate_root=kind.aggregate_root,
-            metadata=kind.metadata,
-            properties={prop.name: export_property(prop, kind) for prop, kind in kind.all_props_with_kind()},
         )
+        if with_metadata:
+            result["metadata"] = kind.metadata
+        if with_properties:
+            result["allow_unknown_props"] = kind.allow_unknown_props
+            result["properties"] = {prop.name: export_property(prop, kind) for prop, kind in kind.all_props_with_kind()}
+        if with_relatives:
+            result["bases"] = kind.bases
+            result["predecessor_kinds"] = kind.predecessor_kinds()
+            result["successor_kinds"] = kind.successor_kinds
+        return result
 
     def export_kind(kind: Kind) -> Json:
         if isinstance(kind, SimpleKind):

resotocore/resotocore/model/model.py

@@ -110,6 +110,7 @@ def resolve(self, model: Dict[str, Kind]) -> Kind:
 
     def meta(self, name: str, clazz: Type[T]) -> Optional[T]:
         meta = self.metadata
+        # noinspection PyUnboundLocalVariable
         return v if meta is not None and (v := meta.get(name)) is not None and isinstance(v, clazz) else None
 
     def meta_get(self, name: str, clazz: Type[T], default: T) -> T:
@@ -255,6 +256,14 @@ def check_valid(self, obj: JsonElement, **kwargs: bool) -> ValidationResult:
         - if obj is not valid and can not be coerced: raise an exception
         """
 
+    @staticmethod
+    def to_json(obj: Kind, **kwargs: Any) -> Json:
+        return obj.as_json(**kwargs)
+
+    @abstractmethod
+    def as_json(self, **kwargs: bool) -> Json:
+        pass
+
     def resolve(self, model: Dict[str, Kind]) -> None:
         pass
 
@@ -383,14 +392,9 @@ def coerce_if_required(self, value: JsonElement, **kwargs: bool) -> Optional[Jso
         """
         return None
 
-    def as_json(self) -> Json:
+    def as_json(self, **kwargs: bool) -> Json:
         return {"fqn": self.fqn, "runtime_kind": self.runtime_kind}
 
-    # noinspection PyUnusedLocal
-    @staticmethod
-    def to_json(obj: SimpleKind, **kw_args: object) -> Json:
-        return obj.as_json()
-
 
 class AnyKind(SimpleKind):
     def __init__(self) -> None:
@@ -456,8 +460,8 @@ def coerce_if_required(self, value: JsonElement, **kwargs: bool) -> Optional[str
         else:
             return json.dumps(value)
 
-    def as_json(self) -> Json:
-        js = super().as_json()
+    def as_json(self, **kwargs: bool) -> Json:
+        js = super().as_json(**kwargs)
         if self.pattern:
             js["pattern"] = self.pattern
         if self.enum:
@@ -521,8 +525,8 @@ def coerce_if_required(self, value: JsonElement, **kwargs: bool) -> Optional[Uni
             except ValueError:
                 return None
 
-    def as_json(self) -> Json:
-        js = super().as_json()
+    def as_json(self, **kwargs: bool) -> Json:
+        js = super().as_json(**kwargs)
         if self.enum:
             js["enum"] = self.enum
         if self.minimum:
@@ -743,7 +747,7 @@ def resolve(self, model: Dict[str, Kind]) -> None:
         else:
             raise AttributeError(f"Underlying kind not known: {self.destination_fqn}")
 
-    def as_json(self) -> Json:
+    def as_json(self, **kwargs: bool) -> Json:
         return {
             "fqn": self.fqn,
             "runtime_kind": self.runtime_kind,
@@ -758,6 +762,9 @@ def __init__(self, inner: Kind):
         super().__init__(f"{inner.fqn}[]")
         self.inner = inner
 
+    def as_json(self, **kwargs: bool) -> Json:
+        return {"fqn": self.fqn, "inner": self.inner.as_json(**kwargs)}
+
     def resolve(self, model: Dict[str, Kind]) -> None:
         self.inner.resolve(model)
 
@@ -805,6 +812,13 @@ def __init__(self, key_kind: Kind, value_kind: Kind):
         self.key_kind = key_kind
         self.value_kind = value_kind
 
+    def as_json(self, **kwargs: bool) -> Json:
+        return {
+            "fqn": self.fqn,
+            "key_kind": self.key_kind.as_json(**kwargs),
+            "value_kind": self.value_kind.as_json(**kwargs),
+        }
+
     def check_valid(self, obj: JsonElement, **kwargs: bool) -> ValidationResult:
         if isinstance(obj, dict):
             coerced = self.coerce_if_required(obj, **kwargs)
@@ -868,6 +882,18 @@ def __init__(
         self.__property_by_path: List[ResolvedProperty] = []
         self.__synthetic_props: List[ResolvedProperty] = []
 
+    def as_json(self, **kwargs: bool) -> Json:
+        result: Json = {"fqn": self.fqn, "aggregate_root": self.aggregate_root}
+        if kwargs.get("with_metadata", True):
+            result["metadata"] = self.metadata
+        if kwargs.get("with_properties", True):
+            result["allow_unknown_props"] = self.allow_unknown_props
+            result["properties"] = [to_js(prop) for prop in self.properties]
+        if kwargs.get("with_relatives", True):
+            result["bases"] = self.bases
+            result["successor_kinds"] = self.successor_kinds
+        return result
+
     def copy(
         self,
         *,
@@ -1569,4 +1595,4 @@ class UsageDatapoint:
 
 # register serializer for this class
 set_deserializer(Kind.from_json, Kind)
-set_serializer(SimpleKind.to_json, SimpleKind)
+set_serializer(Kind.to_json, Kind)

resotocore/resotocore/web/api.py

@@ -855,23 +855,35 @@ async def get_model(self, request: Request, deps: TenantDependencies) -> StreamR
         full_model = await deps.model_handler.load_model(graph_id)
         with_bases = if_set(request.query.get("with_bases"), lambda x: x.lower() == "true", False)
         with_property_kinds = if_set(request.query.get("with_property_kinds"), lambda x: x.lower() == "true", False)
+        with_properties = if_set(request.query.get("with_properties"), lambda x: x.lower() == "true", True)
+        with_relatives = if_set(request.query.get("with_relatives"), lambda x: x.lower() == "true", True)
+        with_metadata = if_set(request.query.get("with_metadata"), lambda x: x.lower() == "true", True)
+        aggregate_roots_only = if_set(request.query.get("aggregate_roots_only"), lambda x: x.lower() == "true", False)
         md = full_model
         if kind := request.query.get("kind"):
             kinds = set(kind.split(","))
             md = md.filter_complex(lambda x: x.fqn in kinds, with_bases, with_property_kinds)
         if filter_names := request.query.get("filter"):
             parts = filter_names.split(",")
             md = md.filter_complex(lambda x: any(x.fqn in p for p in parts), with_bases, with_property_kinds)
+        if aggregate_roots_only:
+            md = md.filter_complex(lambda x: x.aggregate_root, with_bases, with_property_kinds)
         md = md.flat_kinds(full_model) if request.query.get("flat", "false") == "true" else md
 
         export_format = request.query.get("format")
         # default to internal model format, but allow requesting json schema format
         if export_format == "schema" or request.headers.get("accept") == "application/schema+json":
             return json_response(json_schema(md), content_type="application/schema+json")
         elif export_format == "simple":
-            return await single_result(request, json_export_simple_schema(md))
+            return await single_result(
+                request, json_export_simple_schema(md, with_properties, with_relatives, with_metadata)
+            )
         else:
-            return await single_result(request, to_js(md.kinds.values(), strip_nulls=True))
+            json_model = [
+                m.as_json(with_properties=with_properties, with_relatives=with_relatives, with_metadata=with_metadata)
+                for m in md.kinds.values()
+            ]
+            return await single_result(request, json.loads(json.dumps(json_model, sort_keys=True)))
 
     async def update_model(self, request: Request, deps: TenantDependencies) -> StreamResponse:
         graph_id = GraphName(request.match_info.get("graph_id", "resoto"))

resotolib/resotolib/core/model_export.py

@@ -150,6 +150,7 @@ def dataclasses_to_resotocore_model(
     aggregate_root: Optional[Type[Any]] = None,
     walk_subclasses: bool = True,
     use_optional_as_required: bool = False,
+    with_description: bool = True,
 ) -> List[Json]:
     """
     Analyze all transitive dataclasses and create the model
@@ -162,26 +163,33 @@ def dataclasses_to_resotocore_model(
     :param aggregate_root: if a type is a subtype of this type, it will be considered an aggregate root.
     :param walk_subclasses: if true, all subclasses of the given classes will be analyzed as well.
     :param use_optional_as_required: if true, all non-optional fields will be considered required.
+    :param with_description: if true, include the description for classes and properties.
     :return: the model definition in the resotocore json format.
     """
 
     def prop(field: Attribute) -> List[Json]:  # type: ignore
         # the field itself can define the type via a type hint
         # this is useful for int and float in python where the representation can not be
         # detected by the type itself. Example: int32/int64 or float/double
-        # If not defined, we fallback to the largest container: int64 and double
+        # If not defined, we fall back to the largest container: int64 and double
         name = field.name
         meta = field.metadata.copy()
         kind = meta.pop("type_hint", model_name(field.type))
-        desc = meta.pop("description", "")
+        desc = meta.pop("description", None)
+        desc = desc if with_description else None
         required = meta.pop("required", use_optional_as_required and not is_optional(field.type))  # type: ignore
         synthetic = meta.pop("synthetic", None)
         synthetic = synthetic if synthetic else {}
         for ps in property_metadata_to_strip:
             meta.pop(ps, None)
 
         def json(
-            name: str, kind_str: str, required: bool, description: str, meta: Optional[Dict[str, str]], **kwargs: Any
+            name: str,
+            kind_str: str,
+            required: bool,
+            description: Optional[str],
+            meta: Optional[Dict[str, str]],
+            **kwargs: Any,
         ) -> Json:
             js = {"name": name, "kind": kind_str, "required": required, "description": description, **kwargs}
             if meta:
@@ -193,7 +201,7 @@ def json(
                 synth_prop,
                 synth_trafo,
                 False,
-                f"Synthetic prop {synth_trafo} on {name}",
+                None,
                 None,
                 synthetic={"path": [name]},
             )
@@ -236,9 +244,13 @@ def export_data_class(clazz: type) -> None:
         ]
         root = any(sup == aggregate_root for sup in clazz.mro()) if aggregate_root else True
         kind = model_name(clazz)
-        metadata: Optional[Json] = None
+        metadata: Json = {}
         if (m := getattr(clazz, "metadata", None)) and isinstance(m, dict):
-            metadata = m
+            metadata = m.copy()
+        if (s := clazz.__dict__.get("kind_display", None)) and isinstance(s, str):
+            metadata["name"] = s
+        if with_description and (s := clazz.__dict__.get("kind_description", None)) and isinstance(s, str):
+            metadata["description"] = s
 
         model.append(
             {
@@ -248,7 +260,7 @@ def export_data_class(clazz: type) -> None:
                 "allow_unknown_props": allow_unknown_props,
                 "successor_kinds": successors.get(kind, None),
                 "aggregate_root": root,
-                "metadata": metadata,
+                "metadata": metadata or None,
             }
         )
 

resotolib/resotolib/graph/__init__.py

@@ -391,7 +391,7 @@ def export_model(graph: Optional[Graph] = None, **kwargs: Any) -> List[Json]:
         for node in graph.nodes:
             classes.add(type(node))
 
-    model = resource_classes_to_resotocore_model(classes, aggregate_root=BaseResource, **kwargs)
+    model = resource_classes_to_resotocore_model(classes, aggregate_root=BaseResource, with_description=False, **kwargs)
     for resource_model in model:
         if resource_model.get("fqn") == "resource":
             resource_model.get("properties", []).append(