[resotocore][feat] Provide property complete endpoint (#1827)

resotocore/Makefile

@@ -59,7 +59,6 @@ lint: ## static code analysis
 	black --line-length 120 --check resotocore tests
 	flake8 resotocore
 	pylint resotocore
-	# mypy --python-version 3.9 --strict --install-types --non-interactive resotocore tests
 	mypy --python-version 3.9 --strict resotocore tests
 
 test: ## run tests quickly with the default Python

resotocore/resotocore/cli/command.py

@@ -5907,7 +5907,7 @@ async def database_synchronize(
             resoto_model = await self.dependencies.model_handler.load_model(ctx.graph_name)
 
             if complete_model:
-                complex_kinds = resoto_model.complex_kinds()
+                complex_kinds = list(resoto_model.complex_kinds())
                 kinds = list(resoto_model.kinds.values())
             else:
                 # only export the kinds that are exported by this query

resotocore/resotocore/model/model.py

@@ -23,6 +23,7 @@
     Tuple,
     Iterable,
     TypeVar,
+    Iterator,
 )
 
 import yaml
@@ -1352,8 +1353,15 @@ def get(self, name_or_object: Union[str, Json]) -> Optional[Kind]:
         else:
             return None
 
-    def complex_kinds(self) -> List[ComplexKind]:
-        return [k for k in self.kinds.values() if isinstance(k, ComplexKind)]
+    def complex_kinds(self) -> Iterator[ComplexKind]:
+        for k in self.kinds.values():
+            if isinstance(k, ComplexKind):
+                yield k
+
+    def aggregate_roots(self) -> Iterator[ComplexKind]:
+        for k in self.complex_kinds():
+            if k.aggregate_root:
+                yield k
 
     def property_by_path(self, path_: str) -> ResolvedProperty:
         path = PropertyPath.from_path(path_)
@@ -1569,6 +1577,64 @@ def add_kind(cpl: ComplexKind) -> None:
 
         return Model(kinds, self.__property_kind_by_path)
 
+    def complete_path(
+        self,
+        path: str,
+        prop: str,
+        *,
+        filter_kinds: Optional[List[str]] = None,
+        fuzzy: bool = False,
+        skip: int = 0,
+        limit: int = 20,
+    ) -> Tuple[int, Dict[str, str]]:
+        filter_prop: Callable[[str], bool] = (
+            (lambda p: re.match(prop, p) is not None) if fuzzy else (lambda p: p.startswith(prop))
+        )
+
+        def from_prop_path(kinds: Iterable[ComplexKind]) -> Tuple[int, Dict[str, str]]:
+            props: Dict[str, str] = {}
+            for kind in kinds:
+                for p, pk in kind.all_props_with_kind():
+                    if filter_prop(p.name):
+                        if isinstance(pk, ArrayKind):
+                            props[p.name + "[*]"] = pk.inner.fqn
+                        elif isinstance(pk, TransformKind):
+                            props[p.name] = (pk.source_kind or any_kind).fqn
+                        else:
+                            props[p.name] = pk.fqn
+            return len(props), {k: props[k] for k in sorted(props)[skip : skip + limit]}
+
+        def from_aggregate_roots() -> Tuple[int, Dict[str, str]]:
+            roots: Dict[str, str] = {}
+            for kind in self.aggregate_roots():
+                if filter_prop(kind.fqn) and (not filter_kinds or kind.fqn in filter_kinds):
+                    roots[kind.fqn] = "node"
+            return len(roots), {k: roots[k] for k in sorted(roots)[skip : skip + limit]}
+
+        if path:
+            parts = path.split(".")
+            if parts[0] in ("ancestors", "descendants"):
+                if len(parts) == 1:
+                    return from_aggregate_roots()
+                elif len(parts) == 2:
+                    return 1, {"reported": "resource"}
+                elif len(parts) == 3 and parts[2] == "reported" and (kind := self.get(parts[1])):
+                    return from_prop_path([kind]) if isinstance(kind, ComplexKind) else (0, {})
+                elif len(parts) > 3 and parts[2] == "reported":
+                    parts = parts[3:]
+                else:
+                    return 0, {}
+            elif parts[0] in ("reported", "metadata", "desired"):
+                parts = parts[1:]
+            path = ".".join(parts)
+
+        if path:
+            resolved = self.property_by_path(path)
+            return from_prop_path([resolved.kind]) if isinstance(resolved.kind, ComplexKind) else (0, {})
+        else:
+            roots = [k for k in self.aggregate_roots() if not filter_kinds or k.fqn in filter_kinds]
+            return from_prop_path(roots)
+
 
 @frozen
 class UsageDatapoint:

resotocore/resotocore/static/api-doc.yaml

@@ -226,10 +226,10 @@ paths:
           in: query
           schema:
             type: string
-            default: null
             enum:
               - schema
               - simple
+          required: false
         - name: with_property_kinds
           description: "Render types of property values. Only together with kind or filter"
           in: query
@@ -772,7 +772,7 @@ paths:
         - node_management
       parameters:
         - $ref: "#/components/parameters/graph_id"
-        - $ref: "#/components/parameters/section"
+        - $ref: "#/components/parameters/section-path"
         - name: node_id
           in: path
           description: "The identifier of the node"
@@ -1706,6 +1706,75 @@ paths:
                   ]
                 items:
                   type: string
+  /graph/{graph_id}/property/path/complete:
+    post:
+      summary: "Search the graph and return all possible attribute values for given property path."
+      tags:
+        - graph_search
+      parameters:
+        - $ref: "#/components/parameters/graph_id"
+        - $ref: "#/components/parameters/section"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              description: "Parameters to define a property path completion."
+              type: object
+              properties:
+                prop:
+                  type: string
+                  description: |
+                    The last part of the property name used to filter possible results.
+                    All property names that start with this name are returned.
+                    In fuzzy mode, this property is interpreted as regular expression.
+                  example: "ab"
+                path:
+                  type: string
+                  description: Already existing path
+                kinds:
+                  type: array
+                  items:
+                    type: string
+                    description: The list of allowed kinds.
+                fuzzy:
+                  type: boolean
+                  description: |
+                    If true, the prop parameter is interpreted as regular expression.
+                    If false, the prop parameter is interpreted as prefix.
+                  default: false
+                limit:
+                  type: integer
+                  description: |
+                    The maximum number of results to return.
+                    If not defined, the default limit is used.
+                  default: 20
+                skip:
+                  type: integer
+                  description: |
+                    The number of results to skip.
+                    If not defined, the default skip is used.
+                  default: 0
+      responses:
+        "200":
+          description: "The result of this search in the defined format"
+          headers:
+            Total-Count:
+              description: "The number of total available elements"
+              schema:
+                type: integer
+                example: 3
+          content:
+            "application/json":
+              schema:
+                type: array
+                example: |
+                  [
+                    "owner",
+                    "checksum/secret",
+                    "prometheus.io/path",
+                  ]
+                items:
+                  type: string
 
 
 
@@ -3185,6 +3254,17 @@ components:
       required: true
       schema:
         type: string
+    section-path:
+      name: section
+      in: path
+      required: true
+      description: "The name of the section used for all property paths. If not defined root is assumed."
+      schema:
+        type: string
+        enum:
+          - reported
+          - desired
+          - metadata
     section:
       name: section
       in: query

resotocore/resotocore/web/api.py

@@ -90,7 +90,7 @@
 from resotocore.model.exportable_model import json_export_simple_schema
 from resotocore.model.graph_access import Section
 from resotocore.model.json_schema import json_schema
-from resotocore.model.model import Kind
+from resotocore.model.model import Kind, Model
 from resotocore.model.typed_model import to_json, from_js, to_js_str, to_js
 from resotocore.query.model import Predicate, PathRoot, variable_to_absolute
 from resotocore.query.query_parser import predicate_term
@@ -231,6 +231,7 @@ def __add_routes(self, prefix: str) -> None:
                 web.post(prefix + "/graph/{graph_id}/search/history/aggregate", require(self.query_history, r)),
                 web.post(prefix + "/graph/{graph_id}/property/attributes", require(self.possible_values, r)),
                 web.post(prefix + "/graph/{graph_id}/property/values", require(self.possible_values, r)),
+                web.post(prefix + "/graph/{graph_id}/property/path/complete", require(self.property_path_complete, r)),
                 # maintain the graph
                 web.patch(prefix + "/graph/{graph_id}/nodes", require(self.update_nodes, a)),
                 web.post(prefix + "/graph/{graph_id}/merge", require(self.merge_graph, a)),
@@ -1012,11 +1013,7 @@ async def abort_batch(self, request: Request, deps: TenantDependencies) -> Strea
         await graph_db.abort_update(batch_id)
         return web.HTTPOk(body="Batch aborted.")
 
-    async def graph_query_model_from_request(
-        self, request: Request, deps: TenantDependencies
-    ) -> Tuple[GraphDB, QueryModel]:
-        section = section_of(request)
-        query_string = await request.text()
+    async def graph_model_from_request(self, request: Request, deps: TenantDependencies) -> Tuple[GraphName, Model]:
         graph_name = GraphName(request.match_info.get("graph_id", "resoto"))
         raw_at = request.query.get("at")
         at = date_parser.parse(raw_at) if raw_at else None
@@ -1027,10 +1024,17 @@ async def graph_query_model_from_request(
                 raise ValueError(f"No snapshot found for {graph_name} at {at}")
 
         graph_name = snapshot_name or graph_name
+        return graph_name, await deps.model_handler.load_model(graph_name)
+
+    async def graph_query_model_from_request(
+        self, request: Request, deps: TenantDependencies
+    ) -> Tuple[GraphDB, QueryModel]:
+        section = section_of(request)
+        query_string = await request.text()
+        graph_name, model = await self.graph_model_from_request(request, deps)
         graph_db = deps.db_access.get_graph_db(graph_name)
         q = await deps.template_expander.parse_query(query_string, section, **request.query)
-        m = await deps.model_handler.load_model(graph_name)
-        return graph_db, QueryModel(q, m, cast(Dict[str, Any], request.query))
+        return graph_db, QueryModel(q, model, cast(Dict[str, Any], request.query))
 
     async def raw(self, request: Request, deps: TenantDependencies) -> StreamResponse:
         graph_db, query_model = await self.graph_query_model_from_request(request, deps)
@@ -1043,6 +1047,20 @@ async def explain(self, request: Request, deps: TenantDependencies) -> StreamRes
         result = await graph_db.explain(query_model)
         return web.json_response(to_js(result))
 
+    async def property_path_complete(self, request: Request, deps: TenantDependencies) -> StreamResponse:
+        _, model = await self.graph_model_from_request(request, deps)
+        body = await request.json()
+        path = variable_to_absolute(section_of(request), body.get("path", PathRoot)).rstrip(".\n\t")
+        prop = body.get("prop", "")
+        filter_kinds = body.get("kinds")
+        fuzzy = body.get("fuzzy", False)
+        limit = body.get("limit", 20)
+        skip = body.get("skip", 0)
+        assert skip >= 0, "Skip must be positive"
+        assert limit > 0, "Limit must be positive"
+        count, result = model.complete_path(path, prop, filter_kinds=filter_kinds, fuzzy=fuzzy, skip=skip, limit=limit)
+        return await single_result(request, result, {"Total-Count": str(count)})
+
     async def possible_values(self, request: Request, deps: TenantDependencies) -> StreamResponse:
         graph_db, query_model = await self.graph_query_model_from_request(request, deps)
         section = section_of(request)

resotocore/tests/resotocore/model/model_test.py

@@ -582,6 +582,42 @@ def test_filter_model(person_model: Model) -> None:
     assert address.predecessor_kinds() == {"default": ["Person"], "delete": []}
 
 
+def test_complete_path(person_model: Model) -> None:
+    count, all_props = person_model.complete_path("", "", fuzzy=False, skip=0, limit=4)
+    assert count == 21  # total
+    assert all_props == {"address": "Address", "addresses[*]": "Address", "age": "duration", "any": "any"}
+    # filter by prefix
+    count, all_props = person_model.complete_path("", "ad", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"address": "Address", "addresses[*]": "Address"}
+    # filter by prefix fuzzy
+    count, all_props = person_model.complete_path("", "^a.*s", fuzzy=True, skip=0, limit=4)
+    assert all_props == {"address": "Address", "addresses[*]": "Address"}
+    # skip the first 4
+    count, all_props = person_model.complete_path("", "", fuzzy=False, skip=4, limit=4)
+    assert all_props == {"city": "string", "ctime": "datetime", "expires": "datetime", "exported_age": "duration"}
+    # ask for a nested kind
+    count, all_props = person_model.complete_path("address", "", fuzzy=False, skip=4, limit=4)
+    assert all_props == {"mtime": "datetime", "tags": "dictionary[string, string]", "zip": "zip"}
+    # ask for a nested kind filtered
+    count, all_props = person_model.complete_path("address", "ta", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"tags": "dictionary[string, string]"}
+    # reported section
+    count, all_props = person_model.complete_path("reported.address", "ta", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"tags": "dictionary[string, string]"}
+    # ask for ancestors will list all available kinds
+    count, all_props = person_model.complete_path("ancestors", "", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"Address": "node", "Base": "node", "Person": "node", "account": "node"}
+    # suggest the reported section
+    count, all_props = person_model.complete_path("ancestors.Address", "", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"reported": "resource"}
+    # suggest properties of address
+    count, all_props = person_model.complete_path("ancestors.Address.reported", "", fuzzy=False, skip=0, limit=4)
+    assert all_props == {"city": "string", "id": "string", "kind": "string", "list[*]": "string"}
+    # filter by kinds
+    count, all_props = person_model.complete_path("", "", filter_kinds=["Address"], skip=5)
+    assert all_props == {"tags": "dictionary[string, string]", "zip": "zip"}
+
+
 @given(json_object_gen)
 @settings(max_examples=200, suppress_health_check=list(HealthCheck))
 def test_yaml_generation(js: Json) -> None: