Fixed a bunch of filters and stuff

codoc_views/codoc_base.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-from codoc.service.parsing.node import get_identifier_of_object
 
 from codoc.service import filters
 from codoc.service.export.codoc_view import view
@@ -12,19 +11,45 @@
 )
 def view_modules(graph):
     """
-    This view contains all the modules that our system contain.
+
+    This diagram can be a bit verbose, but shows all external dependencies
+    and where we and how we depend on them.
+
+    It is good to get an overview of the whole system and how it connects
+    with the outside world.
     """
-    return filters.exclude_functions(filters.exclude_classes(graph))
+    graph = filters.get_depth_based_filter(2)(graph)
+    return filters.include_only_modules(graph)
 
 
 @view(
     label="Internal Module View of the Codoc SDK",
 )
 def view_modules_internal(graph):
     """
-    This view contains all the modules that our system contain.
+    This view displays the internal structure of the Codoc Python
+    system.
+
+    The main purpose is to see how different elements are inter-dependent.
+
+    The service layer is created to expose pure functionality of the framework.
+
+    We try to develop things in a pure functional fashion without side effects, and
+    general adhere to immutable data types.
+
+    We utilize a service layer with the intent of separating
+    the domain model from usage.
+
+    The top level of the service layer should expose a bunch of functions
+    that can  be used by a given entry point
+    (be it the CLI or when used as a functional framework)
+
+    Our usage of a service layer is heavily inspired by
+    [Architectural Patterns In Python](https://cosmicpython.com).
+
+    We also have a data model. These are pure dataclasses,
+    and are the basis of the overall system.
+
     """
-    graph = filters.get_children_of(
-        get_identifier_of_object(codoc), keep_external_nodes=False
-    )(graph)
-    return filters.exclude_functions(filters.exclude_classes(graph))
+    graph = filters.get_children_of(codoc)(graph)
+    return filters.include_only_modules(graph)

src/codoc/service/dependency_bubble.py → src/codoc/service/dependency_correcting.py

@@ -51,3 +51,21 @@ def _get_set_of_all_parents(identifier: NodeId, graph: Graph) -> Set[Node]:
         return {node}
 
     return _get_set_of_all_parents(parent_id, graph) | {node}
+
+
+def remove_non_connected_edges(graph: Graph) -> Graph:
+    node_identifiers = set(node.identifier for node in graph.nodes)
+    edges = set(
+        edge
+        for edge in graph.edges
+        if is_both_edges_of_edge_in_list_of_nodes(edge, node_identifiers)
+    )
+    return Graph(nodes=graph.nodes, edges=edges)
+
+
+def is_both_edges_of_edge_in_list_of_nodes(
+    edge: Dependency, node_identifiers: Set[str]
+) -> bool:
+    is_from_node_internal = edge.from_node in node_identifiers
+    is_to_node_internal = edge.to_node in node_identifiers
+    return is_from_node_internal and is_to_node_internal

src/codoc/service/export/codoc_api.py

@@ -2,6 +2,7 @@
 import requests
 from typing import Optional, Dict, Any
 from codoc.domain.model import Graph
+from codoc.service.dependency_correcting import remove_non_connected_edges
 import logging
 
 logger = logging.getLogger(__name__)
@@ -25,6 +26,8 @@ def publish(
     """
     if not api_key:
         raise ApiKeyNotSupplied()
+    # TODO move this to somewhere else
+    graph = remove_non_connected_edges(graph)
 
     payload = _get_payload(
         graph=graph,

src/codoc/service/filters/__init__.py

@@ -18,6 +18,7 @@
     include_only_modules,
     include_only_exceptions,
 )
+from .regex_based import filter_by_regex, exclude_by_regex
 
 
 __all__ = [
@@ -33,4 +34,6 @@
     "get_children_of",
     "get_depth_based_filter",
     "exclude_external",
+    "filter_by_regex",
+    "exclude_by_regex",
 ]

src/codoc/service/filters/children_based.py

@@ -31,15 +31,13 @@ def get_children_of(
 
     .. code-block:: python
 
-        from codoc.service.parsing.node import get_identifier_of_object
+       # returns all modules/classes/exceptions/functions
+       # defined inside `myporject.subproject`.
 
-       identifier = get_identifier_of_object()
        filter_function = filters.get_children_of(myproject.subproject)
 
        filtered_graph = filter_function(graph)
 
-
-
     """
     if not isinstance(node, str):
         if isinstance(node, Node):
@@ -67,7 +65,7 @@ def filter_func(graph: Graph) -> Graph:
                     )
                     for node in internal_nodes
                 },
-                edges=edges,
+                edges=graph.edges,
             )
         else:
             # Also remove `parent_id` for all nodes if parent_id is outside.
@@ -79,7 +77,7 @@ def filter_func(graph: Graph) -> Graph:
                     if is_node_in_edges(node, edges, graph)
                     or is_node_accepted(node, graph, identifier)
                 },
-                edges=edges,
+                edges=graph.edges,
             )
 
     return filter_func

src/codoc/service/filters/class_diagram.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 from codoc.domain.model import Graph, Node, NodeType, Dependency
-from codoc.domain.helpers import get_node
-from .helpers import node_without_parent
+from codoc.domain.helpers import get_node, set_parent
 
 
+# TODO fix. this is so fucking wrong
 def class_diagram_filter(graph: Graph) -> Graph:
     """
     :param graph: Graph to filter
@@ -32,7 +32,7 @@ def node_with_non_class_parent(node: Node, graph) -> Node:
     if node.parent_identifier and not identifier_is_class(
         node.parent_identifier, graph
     ):
-        return node_without_parent(node)
+        return set_parent(node, None)
     return node
 
 

src/codoc/service/filters/depth_based.py

@@ -17,16 +17,13 @@
 
 from .types import FilterType
 
-from .helpers import get_edges_where_both_ends_are_in_nodes
-
 
 def get_depth_based_filter(depth: int) -> FilterType:
     def depth_based_filter(graph: Graph) -> Graph:
         nodes = set(
             node for node in graph.nodes if get_depth_of_node(node, graph) <= depth
         )
-        edges = get_edges_where_both_ends_are_in_nodes(graph.edges, nodes)
-        return Graph(nodes=nodes, edges=edges)
+        return Graph(nodes=nodes, edges=graph.edges)
 
     return depth_based_filter
 

src/codoc/service/filters/external_exclusion.py

@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 
 from codoc.domain.model import Graph
+from .high_order_filter import filter_nodes_by_lambda
 
-from .helpers import get_edges_where_both_ends_are_in_nodes
 
-
-# TODO this fails if you do an OR on two graphs.
+# TODO this fails if you do an OR on two graphs where node is in both.
 def exclude_external(graph: Graph) -> Graph:
-    nodes = {node for node in graph.nodes if not node.external}
-    edges = get_edges_where_both_ends_are_in_nodes(graph.edges, nodes)
-
-    return Graph(nodes=nodes, edges=edges)
+    return filter_nodes_by_lambda(lambda node: not node.external)(graph)

src/codoc/service/filters/helpers.py

@@ -4,12 +4,6 @@
 from codoc.domain.model import Dependency, Node
 
 
-def node_without_parent(node: Node) -> Node:
-    kwargs = node.__dict__
-    kwargs["parent_identifier"] = None
-    return Node(**kwargs)
-
-
 def get_edges_where_both_ends_are_in_nodes(edges: Set[Dependency], nodes: Set[Node]):
     node_identifiers = set(node.identifier for node in nodes)
     return set(

src/codoc/service/filters/high_order_filter.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+from typing import Callable
+
+from codoc.domain.model import Graph, Node
+
+from .types import FilterType
+
+
+def filter_nodes_by_lambda(filter_function: Callable[[Node], bool]) -> FilterType:
+    def internal_filter(graph: Graph) -> Graph:
+        nodes = {node for node in graph.nodes if filter_function(node)}
+        return Graph(nodes=nodes, edges=graph.edges)
+
+    return internal_filter

src/codoc/service/filters/regex_based.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import re
+
+from .high_order_filter import filter_nodes_by_lambda
+from .types import FilterType
+
+
+def filter_by_regex(pattern: str, flags=0) -> FilterType:
+    """
+    This allows you to filter nodes based on whether they fulfill
+    some regex query. This is ideal if you, for instance, want to remove all
+    test related things.
+
+    The regex is done solely on the `name` attribute.
+
+    The following example removes all instances of "test".
+    Example:
+       graph = filters.filter_by_regex("test", flags=re.IGNORECASE)(graph)
+
+    To understand how to use regex, please consult the python documentation:
+
+    https://docs.python.org/3/library/re.html
+    """
+
+    pattern = re.compile(pattern, flags=flags)
+    return filter_nodes_by_lambda(lambda node: pattern.search(node.name) is not None)
+
+
+def exclude_by_regex(pattern: str, flags=0) -> FilterType:
+    """
+    This allows you to filter nodes based on whether they fulfill
+    some regex query. This is ideal if you, for instance, want to remove all
+    test related things.
+
+    The regex is done solely on the `name` attribute.
+
+    The following example removes all instances of "test".
+    Example:
+       graph = filters.filter_by_regex("test", flags=re.IGNORECASE)(graph)
+
+    To understand how to use regex, please consult the python documentation:
+
+    https://docs.python.org/3/library/re.html
+    """
+
+    pattern = re.compile(pattern, flags=flags)
+    return filter_nodes_by_lambda(lambda node: pattern.search(node.name) is None)

src/codoc/service/filters/type_exclusion_filter.py

@@ -6,70 +6,72 @@
 
 i.e exclude_modules will return a new graph with all module nodes have been removed.
 """
-from codoc.domain.model import Graph, Node, NodeType, Dependency
-from codoc.domain.helpers import get_node
-from .helpers import node_without_parent
+from codoc.domain.helpers import get_node, set_parent
+from codoc.domain.model import Graph, Node, NodeType
+
+# TODO we need to remove invalid edges here somehwere.
+# Maybe we remove shitty edges at the end.
 
 
 def include_only_modules(graph: Graph) -> Graph:
     """
     Returns a graph that only has modules
     """
-    return TypeBasedFilter(NodeType.MODULE, exclusive=False).exclude(graph)
+    return TypeBasedFilter(NodeType.MODULE, exclusive=False).filter(graph)
 
 
 def include_only_functions(graph: Graph) -> Graph:
     """
     Returns a graph that only has functions
     """
-    return TypeBasedFilter(NodeType.FUNCTION, exclusive=False).exclude(graph)
+    return TypeBasedFilter(NodeType.FUNCTION, exclusive=False).filter(graph)
 
 
 def include_only_classes(graph: Graph) -> Graph:
     """
     Returns a graph that only has classes
     """
-    return TypeBasedFilter(NodeType.CLASS, exclusive=False).exclude(graph)
+    return TypeBasedFilter(NodeType.CLASS, exclusive=False).filter(graph)
 
 
 def include_only_exceptions(graph: Graph) -> Graph:
     """
     Returns a graph that only has exceptions
     """
-    return TypeBasedFilter(NodeType.EXCEPTION, exclusive=False).exclude(graph)
+    return TypeBasedFilter(NodeType.EXCEPTION, exclusive=False).filter(graph)
 
 
 def exclude_modules(graph: Graph) -> Graph:
     """
     Returns a graph that doesn't have any modules
     """
-    return TypeBasedFilter(NodeType.MODULE).exclude(graph)
+    return TypeBasedFilter(NodeType.MODULE).filter(graph)
 
 
 def exclude_functions(graph: Graph) -> Graph:
     """
     Returns a graph that doesn't have any functions
     """
-    return TypeBasedFilter(NodeType.FUNCTION).exclude(graph)
+    return TypeBasedFilter(NodeType.FUNCTION).filter(graph)
 
 
 def exclude_classes(graph: Graph) -> Graph:
     """
     Returns a graph that doesn't have any classes
     """
-    return TypeBasedFilter(NodeType.CLASS).exclude(graph)
+    return TypeBasedFilter(NodeType.CLASS).filter(graph)
 
 
 def exclude_exceptions(graph: Graph) -> Graph:
     """
     Returns a graph that doesn't have any exceptions
     """
-    return TypeBasedFilter(NodeType.EXCEPTION).exclude(graph)
+    return TypeBasedFilter(NodeType.EXCEPTION).filter(graph)
 
 
 class TypeBasedFilter:
     """
-    Filte
+    Filters graphs based on the defined type.
     """
 
     # TODO make this inclusive too
@@ -78,26 +80,21 @@ def __init__(self, based_on_type: NodeType, exclusive: bool = True):
         self._type = based_on_type
         self._exclusive = exclusive
 
-    def exclude(self, graph: Graph) -> Graph:
+    def filter(self, graph: Graph) -> Graph:
         return Graph(
-            edges=set(edge for edge in graph.edges if self.edge_permitted(edge, graph)),
+            edges=graph.edges,
             nodes=set(
                 self.node_without_parent_of_type(node, graph)
                 for node in graph.nodes
                 if self.permitted(node)
             ),
         )
 
-    def edge_permitted(self, edge: Dependency, graph: Graph) -> bool:
-        return self.identifier_is_permitted(
-            edge.to_node, graph
-        ) and self.identifier_is_permitted(edge.from_node, graph)
-
     def node_without_parent_of_type(self, node: Node, graph) -> Node:
         if node.parent_identifier and self.identifier_is_permitted(
             node.parent_identifier, graph
         ):
-            return node_without_parent(node)
+            return set_parent(node, None)
         return node
 
     def identifier_is_permitted(self, identifier: str, graph) -> bool: