From c52a43c234817129d65c8bcd63976035a5bfe026 Mon Sep 17 00:00:00 2001 From: Casper Weiss Bang Date: Mon, 3 May 2021 08:52:56 +0200 Subject: [PATCH] Update all views --- codoc_views/codoc_base.py | 13 +++- codoc_views/codoc_domain.py | 4 +- codoc_views/codoc_service_layer.py | 73 +++++++++++++++++-- src/codoc/domain/helpers.py | 8 +- src/codoc/service/export/codoc_api.py | 11 +-- src/codoc/service/export/codoc_view.py | 60 +++++++++------ src/codoc/service/filters/children_based.py | 53 +++++++------- .../service/filters/type_exclusion_filter.py | 17 +---- src/codoc/service/finder/views.py | 25 +++++-- src/codoc/service/graph.py | 70 +++++++++++++++++- .../snapshots/snap_test_publishing.py | 4 +- tests/unit/test_filters.py | 6 -- 12 files changed, 245 insertions(+), 99 deletions(-) diff --git a/codoc_views/codoc_base.py b/codoc_views/codoc_base.py index f0c78b5..2c8b318 100644 --- a/codoc_views/codoc_base.py +++ b/codoc_views/codoc_base.py @@ -51,5 +51,14 @@ def view_modules_internal(graph): and are the basis of the overall system. """ - graph = filters.get_children_of(codoc)(graph) - return filters.include_only_modules(graph) + return filters.get_depth_based_filter(2)(filters.get_children_of(codoc)(graph)) + + +@view( + label="Internal(detailed) Module View of the Codoc SDK", + description=view_modules_internal.__docs__, +) +def view_modules_internal_detailed(graph): + return filters.include_only_modules( + filters.get_depth_based_filter(3)(filters.get_children_of(codoc)(graph)) + ) diff --git a/codoc_views/codoc_domain.py b/codoc_views/codoc_domain.py index 12f6969..1637c1a 100644 --- a/codoc_views/codoc_domain.py +++ b/codoc_views/codoc_domain.py @@ -13,7 +13,7 @@ def domain_model(graph): """ This view presents the basic domain model of the - [Codoc](https://codoc.org/) system. + Codoc Python system. It shows the core models. @@ -23,7 +23,7 @@ def domain_model(graph): get_identifier_of_object(codoc.domain.model), keep_external_nodes=False )(graph) - return filters.class_diagram_filter(graph) + return filters.include_only_classes(graph) @view( diff --git a/codoc_views/codoc_service_layer.py b/codoc_views/codoc_service_layer.py index 3f17c0a..d363693 100644 --- a/codoc_views/codoc_service_layer.py +++ b/codoc_views/codoc_service_layer.py @@ -4,7 +4,9 @@ from codoc.service import filters from codoc.service.export.codoc_view import view +import codoc import codoc.service +import codoc.service.export @view( @@ -18,9 +20,7 @@ + get_description(codoc.service), ) def view_parsing(graph): - return filters.exclude_modules( - filters.get_children_of(get_identifier_of_object(codoc.service.parsing))(graph) - ) + return filters.get_children_of(codoc.service.parsing)(graph) @view( @@ -32,10 +32,19 @@ def view_graph_rendering(graph): as well as the direct dependencies of the internal aspects. """ - return filters.exclude_modules( - filters.get_children_of( - get_identifier_of_object(codoc.service.graph), keep_external_nodes=True - )(graph) + # TODO makes sure the order of filters doesn't matter when looking at children. + return filters.get_children_of(codoc.service.graph, keep_external_nodes=True)( + filters.get_children_of(codoc)(graph) + ) + + +@view( + label="Overview of the Service layer", + description=get_description(codoc.service), +) +def service_layer_view(graph): + return filters.get_depth_based_filter(2)( + filters.get_children_of(codoc.service, keep_external_nodes=True)(graph) ) @@ -49,3 +58,53 @@ def view_cli_dependencies(graph): return filters.get_children_of( get_identifier_of_object(codoc.entrypoints), keep_external_nodes=True )(graph) + + +@view( + label="Dependencies of the View decorator", +) +def depdencies_of_view_decorator(graph): + """ + To minimize the bootstrapping of each view, we chose to create view + decorators that make it easy to add relevant details to the specific + view function but also make it easy for the Codoc Python framework + to find any defined view function. + + The current version of the view decorator is closely knitted to the + Codoc API, however, we plan to refactor this to make it simply + return a view, rather than publish a view. + + The view decorator overrides the __name__, __docs__ attributes but also sets label, + description, and graph_id on the function before returning it. + + It also changes the return type to simply return a success or failure regarding + whether the publishing step succeeded. Lastly, it creates a __is_codoc_view attribute, + with the True value. This can then be used when Codoc Python searches for + views by checking objects whether it has that attribute. + + def is_a_codoc_view(obj: object) -> bool: + return getattr(obj, "__is_codoc_view", False) + """ + + return filters.get_children_of( + codoc.service.export.codoc_view, keep_external_nodes=True + )(filters.get_children_of(codoc)(graph)) + + +@view( + label="Context of codoc.service.finder", +) +def view_context_of_finder_module(graph): + """ + This graph showcases the inter dependencies of the graph, + as well as the direct dependencies of + the internal aspects. + """ + finder_graph = filters.get_children_of( + codoc.service.finder, keep_external_nodes=True + )(filters.get_children_of(codoc)(graph)) + service_layer_graph = filters.get_depth_based_filter(2)( + filters.get_children_of(codoc.service, keep_external_nodes=True)(graph) + ) + + return finder_graph | service_layer_graph diff --git a/src/codoc/domain/helpers.py b/src/codoc/domain/helpers.py index 714243c..9ce45de 100644 --- a/src/codoc/domain/helpers.py +++ b/src/codoc/domain/helpers.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from typing import Optional, Set +from typing import Optional, Set, Union import dataclasses from .model import Graph, Node, NodeId @@ -59,3 +59,9 @@ def get_parent_node(current_node: Node, graph: Graph) -> Optional[Node]: return get_node(current_node.parent_identifier, graph) except NodeIdentifierNotFoundException: raise ParentNotFoundException(current_node, graph) + + +def get_identifiers(nodes: Union[Graph, Set[Node]]) -> Set[str]: + if isinstance(nodes, Graph): + nodes = nodes.nodes + return {node.identifier for node in nodes} diff --git a/src/codoc/service/export/codoc_api.py b/src/codoc/service/export/codoc_api.py index 01a1093..57bfbb6 100644 --- a/src/codoc/service/export/codoc_api.py +++ b/src/codoc/service/export/codoc_api.py @@ -2,7 +2,7 @@ import requests from typing import Optional, Dict, Any from codoc.domain.model import Graph -from codoc.service.dependency_correcting import remove_non_connected_edges +from codoc.service.graph import clean_graph import logging logger = logging.getLogger(__name__) @@ -27,7 +27,7 @@ def publish( if not api_key: raise ApiKeyNotSupplied() # TODO move this to somewhere else - graph = remove_non_connected_edges(graph) + graph = clean_graph(graph) payload = _get_payload( graph=graph, @@ -45,7 +45,7 @@ def publish( ) if not resp.ok: - raise PublishFailed(graph_id, resp.text) + raise PublishFailed(graph_id, resp) # TODO should return a URL. ressource = resp.json()["pk"] @@ -102,7 +102,8 @@ class ApiKeyNotSupplied(ExportError): class PublishFailed(ExportError): - def __init__(self, graph_id: str, resp: str): - super().__init__(f"Publishing of {graph_id} failed.\nReason={resp}") + def __init__(self, graph_id: str, resp): + reason = resp.json()["message"] + super().__init__(f"Publishing of {graph_id} failed.\n{reason=}") ... diff --git a/src/codoc/service/export/codoc_view.py b/src/codoc/service/export/codoc_view.py index 3a08914..5d8b0b1 100644 --- a/src/codoc/service/export/codoc_view.py +++ b/src/codoc/service/export/codoc_view.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -from codoc.service.parsing.node import get_name, get_description +from typing import Callable, Optional from codoc.domain.model import Graph -from typing import Optional, Callable from codoc.service.export import publish +from codoc.service.parsing.node import get_description, get_name def view( @@ -12,50 +12,64 @@ def view( graph_id: str = None, description: str = None, ): - def decorator(func): + def decorator(initial_view_function): nonlocal description nonlocal graph_id if description is None: - description = get_description(func) + description = get_description(initial_view_function) if graph_id is None: - graph_id = get_id(func) + graph_id = get_id(initial_view_function) - def decorated_function( + def view_function( graph: Graph, api_key: str, publish_func: Optional[Callable] = None, ): - filtered_graph = func(graph) - if not isinstance(filtered_graph, Graph): - raise ValueError("Return type of '{label}' is not a Graph") + filtered_graph = initial_view_function(graph) + raise_if_graph_is_invalid(filtered_graph, label) + if not publish_func: publish_func = publish - - if len(filtered_graph.nodes) == 0: - raise ValueError(f"Graph '{label}' has no nodes") - + # TODO remove this from the view. this should + # be on the CLI, and not closely knitted. return publish_func( graph=filtered_graph, label=label, graph_id=graph_id, - description=get_description(func), + description=get_description(initial_view_function), api_key=api_key, ) - decorated_function.__name__ = graph_id + view_function = _set_view_function_attributes( + view_function, graph_id=graph_id, label=label, description=description + ) - decorated_function.__docs__ = description - decorated_function.__is_codoc_view = True + return view_function - # Attributes for logging etc - decorated_function.graph_id = graph_id - decorated_function.label = label - decorated_function.description = description + return decorator - return decorated_function - return decorator +def raise_if_graph_is_invalid(graph, label): + if not isinstance(graph, Graph): + raise ValueError(f"Return type of '{label}' is not a Graph") + + if len(graph.nodes) == 0: + raise ValueError(f"Graph '{label}' has no nodes") + + +def _set_view_function_attributes(view_function, graph_id, label, description): + view_function.__name__ = graph_id + + view_function.__docs__ = description + view_function.__is_codoc_view = True + + # Attributes for logging etc + view_function.graph_id = graph_id + view_function.label = label + view_function.description = description + + return view_function def get_id(func) -> str: diff --git a/src/codoc/service/filters/children_based.py b/src/codoc/service/filters/children_based.py index 7de9253..740ebfa 100644 --- a/src/codoc/service/filters/children_based.py +++ b/src/codoc/service/filters/children_based.py @@ -1,17 +1,22 @@ # /usr/bin/env python3 from typing import Set, Union -from codoc.domain.model import Graph, Node, NodeId, Dependency + +from codoc.domain.helpers import get_children, get_identifiers, get_node +from codoc.domain.model import Dependency, Graph, Node, NodeId from codoc.service.parsing.node import get_identifier_of_object -from codoc.domain.helpers import get_node, get_children, set_parent -from .types import FilterType + from .helpers import ( is_both_edges_of_edge_in_list_of_nodes, is_either_edges_of_edge_in_list_of_nodes, ) +from .types import FilterType +# TODO addd a lot more tests to this. def get_children_of( - node: Union[str, object, Node], keep_external_nodes: bool = False + node: Union[str, object, Node], + keep_external_nodes: bool = False, + keep_parents: bool = False, ) -> FilterType: """ :param node: The node, object or string identifier of what to filter based on @@ -51,7 +56,7 @@ def filter_func(graph: Graph) -> Graph: internal_nodes = { node for node in graph.nodes if is_node_accepted(node, graph, identifier) } - internal_node_identifiers = {node.identifier for node in internal_nodes} + internal_node_identifiers = get_identifiers(internal_nodes) edges = { edge for edge in graph.edges @@ -59,20 +64,13 @@ def filter_func(graph: Graph) -> Graph: } if not keep_external_nodes: return Graph( - nodes={ - remove_parent_if_parent_is_discarded( - node, internal_node_identifiers - ) - for node in internal_nodes - }, + nodes=internal_nodes, edges=graph.edges, ) else: - # Also remove `parent_id` for all nodes if parent_id is outside. - # TODO keep parent if child is in graph return Graph( nodes={ - remove_parent_if_parent_is_discarded(node, internal_nodes) + node for node in graph.nodes if is_node_in_edges(node, edges, graph) or is_node_accepted(node, graph, identifier) @@ -83,28 +81,31 @@ def filter_func(graph: Graph) -> Graph: return filter_func -def remove_parent_if_parent_is_discarded( - node: Node, node_identifiers: Set[str] -) -> Node: - parent_id = node.parent_identifier - if parent_id is not None and parent_id not in node_identifiers: - return set_parent(node, None) - return node - - # TODO find a way to cache result, i.e dynamic programming # TODO test these functions individually def is_node_accepted(node: Node, graph: Graph, allowed_identifier: NodeId) -> bool: - if node.identifier == allowed_identifier: - return True + return ( + node.identifier == allowed_identifier + or is_parent_accepted(node, graph, allowed_identifier) + or are_children_accepted(node, graph, allowed_identifier) + ) + +def is_parent_accepted(node: Node, graph: Graph, allowed_identifier: NodeId) -> bool: parent_id = node.parent_identifier if not parent_id: return False + if parent_id == allowed_identifier: + return True parent = get_node(node.parent_identifier, graph) - return is_node_accepted(parent, graph, allowed_identifier) + return is_parent_accepted(parent, graph, allowed_identifier) + + +# TODO +def are_children_accepted(node: Node, graph: Graph, allowed_identifier: NodeId) -> bool: + return False def is_edge_accepted( diff --git a/src/codoc/service/filters/type_exclusion_filter.py b/src/codoc/service/filters/type_exclusion_filter.py index 7e54f3b..648a033 100644 --- a/src/codoc/service/filters/type_exclusion_filter.py +++ b/src/codoc/service/filters/type_exclusion_filter.py @@ -6,7 +6,6 @@ i.e exclude_modules will return a new graph with all module nodes have been removed. """ -from codoc.domain.helpers import get_node, set_parent from codoc.domain.model import Graph, Node, NodeType # TODO we need to remove invalid edges here somehwere. @@ -83,22 +82,8 @@ def __init__(self, based_on_type: NodeType, exclusive: bool = True): def filter(self, graph: Graph) -> Graph: return Graph( edges=graph.edges, - nodes=set( - self.node_without_parent_of_type(node, graph) - for node in graph.nodes - if self.permitted(node) - ), + nodes=set(node for node in graph.nodes if self.permitted(node)), ) - def node_without_parent_of_type(self, node: Node, graph) -> Node: - if node.parent_identifier and self.identifier_is_permitted( - node.parent_identifier, graph - ): - return set_parent(node, None) - return node - - def identifier_is_permitted(self, identifier: str, graph) -> bool: - return self.permitted(get_node(identifier, graph)) - def permitted(self, node: Node) -> bool: return (node.of_type is self._type) ^ self._exclusive diff --git a/src/codoc/service/finder/views.py b/src/codoc/service/finder/views.py index 709a035..ca3a1cc 100644 --- a/src/codoc/service/finder/views.py +++ b/src/codoc/service/finder/views.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 from typing import Callable, List +from types import ModuleType import importlib.util import inspect from pathlib import Path @@ -12,21 +13,29 @@ def get_views_in_file(py_file: Path) -> List[CodocView]: - file_name = py_file.name[:-3] - # TODO create a helper for importing shit - spec = importlib.util.spec_from_file_location(file_name, py_file) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + module = get_module_from_file(py_file) views = [ - value - for key, value in inspect.getmembers(module) - if getattr(value, "__is_codoc_view", False) + value for key, value in inspect.getmembers(module) if is_a_codoc_view(value) ] return views +def is_a_codoc_view(obj: object) -> bool: + return getattr(obj, "__is_codoc_view", False) + + +def get_module_from_file(py_file: Path) -> ModuleType: + file_name = py_file.name[:-3] + + spec = importlib.util.spec_from_file_location(file_name, py_file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + def get_views_in_folder(folder: Path) -> List[CodocView]: files = get_all_python_files(folder) return [view for f in files for view in get_views_in_file(f)] diff --git a/src/codoc/service/graph.py b/src/codoc/service/graph.py index 7886a89..cd04d45 100644 --- a/src/codoc/service/graph.py +++ b/src/codoc/service/graph.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 import types +from typing import Set -from codoc.domain.model import Dependency, Graph +from codoc.domain.model import Dependency, Graph, Node +from codoc.domain.helpers import get_identifiers, set_parent +from codoc.service.dependency_correcting import remove_non_connected_edges from codoc.service.parsing.dependency import ( get_dependency_nodes, get_dependency_nodes_with_parents, @@ -41,3 +44,68 @@ def create_graph_of_module( ) ) return create_bubbled_dependencies(Graph(nodes=nodes, edges=edges)) + + +# TODO move +def assert_is_graph_valid(graph: Graph) -> bool: + assert _is_graph(graph), f"{type(graph)=} is not a graph" + assert not _is_graph_empty(graph), "The graph is empty!" + _assert_does_all_parents_exist(graph) + # assert _does_edges_lead_somewhere(graph), "Not all edges lead to existing nodes!" + + +def _is_graph(graph: Graph) -> bool: + return isinstance(graph, Graph) + + +def _assert_does_all_parents_exist(graph: Graph) -> bool: + node_identifiers = get_identifiers(graph) + parentless_nodes = set( + (node.identifier, node.parent_identifier) + for node in graph.nodes + if node.parent_identifier is not None + and node.parent_identifier not in node_identifiers + ) + assert ( + parentless_nodes == set() + ), f"The following parents could not be found: {parentless_nodes}" + + +def _does_edges_lead_somewhere(graph: Graph) -> bool: + node_identifiers = get_identifiers(graph) + return all( + edge.from_node in node_identifiers and edge.to_node in node_identifiers + for edge in graph.edges + ) + ... + + +def _is_graph_empty(graph: Graph) -> bool: + return len(graph.nodes) == 0 + + +# TODO move +def clean_graph(graph: Graph) -> bool: + return remove_parents_if_parent_is_discarded_in_nodes( + remove_non_connected_edges(graph) + ) + + +def remove_parents_if_parent_is_discarded_in_nodes(graph: Graph): + identifiers = get_identifiers(graph) + return Graph( + nodes={ + remove_parent_if_parent_is_discarded(node, identifiers) + for node in graph.nodes + }, + edges=graph.edges, + ) + + +def remove_parent_if_parent_is_discarded( + node: Node, node_identifiers: Set[str] +) -> Node: + parent_id = node.parent_identifier + if parent_id is not None and parent_id not in node_identifiers: + return set_parent(node, None) + return node diff --git a/tests/integration/snapshots/snap_test_publishing.py b/tests/integration/snapshots/snap_test_publishing.py index a3ea150..31710f4 100644 --- a/tests/integration/snapshots/snap_test_publishing.py +++ b/tests/integration/snapshots/snap_test_publishing.py @@ -23,14 +23,14 @@ "nodes": [ { "description": "test", - "identifier": "A", + "identifier": "B", "name": "test", "of_type": "CLASS", "parent_node": None, }, { "description": "test", - "identifier": "B", + "identifier": "A", "name": "test", "of_type": "CLASS", "parent_node": None, diff --git a/tests/unit/test_filters.py b/tests/unit/test_filters.py index 5c55ebb..5ab7723 100644 --- a/tests/unit/test_filters.py +++ b/tests/unit/test_filters.py @@ -117,12 +117,6 @@ def test_removes_parent(self, parent, filtered_graph): def test_keeps_child(self, child, filtered_graph): assert child in filtered_graph.nodes - def test_removes_childs_parent_attribute(self, filtered_graph): - # there should only be the one node - assert len(filtered_graph.nodes) == 1 - child = list(filtered_graph.nodes)[0] - assert child.parent_identifier is None - @pytest.fixture() def filtered_graph(self, parent, child, graph): return filters.get_children_of(child.identifier)(graph)