Merge pull request #3 from svadilfare/strict-mode

Strict mode enabled

codoc_views/config.py

@@ -6,6 +6,6 @@
 import codoc
 
 
-def bootstrap():
+def bootstrap(**kwargs):
     load_dotenv()
-    return create_graph_of_module(codoc)
+    return create_graph_of_module(codoc, **kwargs)

docs/faq.rst

@@ -26,23 +26,64 @@ In the future, it will also make it possible to include information regarding
 the path your code takes when running tests.
 
 
+.. _side_effects:
 
-It crashed!
+Dangerous side effects!
 ---------------------------
 
-That isn't really a question but okay.
+Codocpy relies on dynamic analysis (see :ref:`dynamic_analysis`), which is both
+good and bad. We strongly advise that you don't have any production api keys or
+anything set up, in the environment you run codoc in. Codoc is much like
+automated tests. If your tests executes code that sends emails, then codoc might
+do it too. It's a bit different, but codoc will import all your files into
+memory, and if your code is written improperly, then that means side effects.
+
+This can happen if codocpy imports a file that doesn't
+`define a __main__ function <https://realpython.com/python-main-function/>`_  correctly, and then
+either exits or something similar. Essentially, if you have a python file
+which executes python code on import, then dont run codoc. Rewrite your code.
+It's bad practice.
+
+**TL;DR**
+Do this
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    # scripts/myfile.py
+    if __name__ == "__main__":
+        users = get_users()
+        for user in users
+            send_spam_email(user)
+
+Not this
+~~~~~~~~~~~~~~~~~~~~~~~
+.. warning:: DONT DO THE FOLLOWING
+.. code-block:: bash
+
+    # scripts/myfile.py
+    users = get_users()
+    for user in users
+        send_spam_email(user)
+
+.. _it_crashed:
+
+It crashed!
+---------------------------
+This might be due to the quality of your code, and I mean that in the nicest way
+possible.
 
 Codocpy relies on dynamic analysis (see :ref:`dynamic_analysis`), which means
 that if your code crashes, then codoc crashes. There can be a bunch of different
 reasons. We recommend you read :ref:`prep_env` and make sure it is setup correctly.
 
-Another possible problem is side-effects in your code base. This can happen if
-codocpy imports a file that doesn't `define a __main__ function <https://realpython.com/python-main-function/>`_  correctly, and then
-either exits or something similar. This can happen if you have a python file
-which executes python code on import. Don't do that :)
+You can run codocpy with the ``raise_errors`` for more information, if the error
+message isn't helpful. (``codocpy publish --raise_errors``).
+
+Another possible problem is side-effects in your code base. See :ref:`side_effects`.
 
-If you have circular dependencies, that will make codocpy crash too, due to
-python not handling it.
+If you have circular dependencies, that will make codocpy crash some times, due
+to python crashing.
 
 We try our best at providing meaningful messages, where possible, however it
 might be difficult at times. Codoc is a sensitive framework, but it will help

docs/getting_started.rst

@@ -53,7 +53,6 @@ Inside this folder, create a new file called ``codoc_sample.py``:
         return filters.exclude_functions(filters.exclude_classes(graph))
 
 
-.. note:: All codoc view files have to be prefixed with ``codoc_``
 
 .. _`simple_config`:
 .. _`first_config`:
@@ -70,8 +69,8 @@ You will also need a basic config file in the same folder, called
 
     import myproject
 
-    def bootstrap():
-        return create_graph_of_module(myproject)
+    def bootstrap(**kwargs):
+        return create_graph_of_module(myproject, **kwargs)
 
 .. note:: Using django? Please see :ref:`django` to bootstrap that correctly.
 
@@ -88,6 +87,10 @@ You can verify that codoc can find your views:
 Publishing your view
 ----------------------------------------------------------
 
+.. warning:: Codoc will load all your code, and by effect execute all
+             side-effects! Make sure you don't have files that execute critical
+             code on import! see :ref:`side_effects` for more info.
+
 By now we hope you are already `signed up
 <https://codoc.org/signup/?utm_source=readthedocs&utm_medium=post&utm_campaign=info>`_
 and a registered user.
@@ -115,6 +118,8 @@ You can now publish your views:
     published at https://codoc.org/app/view/181
 
 
+.. note:: Did it failed? Codoc is a bit sensitive, sadly. Read :ref:`it_crashed`
+          for what to do.
 
 Your view is now published, and you can view at the returned domain (in our
 example https://codoc.org/app/graph/181) which shows a public example from our

docs/reference/config.rst

@@ -34,9 +34,9 @@ versions could be by utilizing :ref:`filters`, i.e:
 
     import myproject
 
-    def bootstrap():
+    def bootstrap(**kwargs):
         graph = create_graph_of_module(myproject)
-        return filters.exclude_functions(graph**
+        return filters.exclude_functions(graph, **kwargs)
 
 
 .. _prep_env:
@@ -51,7 +51,7 @@ a need to bootstrap your code before it can run.
 .. _dotenv:
 
 Python dotenv
----------
+-------------
 
 We personally like `python-dotenv <https://pypi.org/project/python-dotenv/>`_,
 and it can easily be used for, for instance, your CODOC API key. Simply add it like so:
@@ -64,9 +64,9 @@ and it can easily be used for, for instance, your CODOC API key. Simply add it l
     import myproj
 
 
-    def bootstrap():
+    def bootstrap(**kwargs):
         load_dotenv()
-        return create_graph_of_module(myproj)
+        return create_graph_of_module(myproj, (**kwargs)
 .. _django:
 
 Django
@@ -84,15 +84,15 @@ backend, which is also written in django:
     from codoc.service.graph import create_graph_of_module
 
 
-    def bootstrap():
+    def bootstrap(**kwargs):
 
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "codoc_api.settings")
         import django
 
         django.setup()
         import organizations
 
-        return create_graph_of_module(organizations)
+        return create_graph_of_module(organizations, **kwargs)
 
 
 An important note with django is that you don't, in the same fashion, have a
@@ -105,7 +105,7 @@ can be combined. We do something like this, in our application:
     from codoc.service.graph import create_graph_of_module
 
 
-    def bootstrap():
+    def bootstrap(**kwargs):
 
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "codoc_api.settings")
         import django
@@ -115,6 +115,7 @@ can be combined. We do something like this, in our application:
         import graphs
 
 
-        return create_graph_of_module(organizations) | create_graph_of_module(graphs)
+        return create_graph_of_module(organizations, **kwargs) |
+        create_graph_of_module(graphs ,**kwargs)
 
 And you can then add all the modules you have that are relevant.

requirements-test.txt

@@ -7,6 +7,6 @@ pytest-mypy==0.8.0
 pytest-picked==0.4.6
 pytest-sugar==0.9.4
 pytest-watch==4.2.0
-python-dotenv==0.15.0
+python-dotenv==0.17.0
 snapshottest==0.6.0
 sphinx==3.0.0

src/codoc/domain/model.py

@@ -72,7 +72,6 @@ def __hash__(self) -> int:
 @dataclass(frozen=True)
 class Graph:
     """
-
     A Graph is the base element of the system.
     It contains both edges (Dependencies) as well as nodes (classes, functions, etc).
 

src/codoc/entrypoints/cli.py

@@ -62,9 +62,13 @@ def __init__(
         if report_errors:
             _setup_sentry()
 
-    def publish(self):
+    def publish(self, strict_mode: bool = False):
         """
         Publish all graphs in the current package
+
+        Args:
+            strict_mode (bool): Whether to terminate if dependency cannot be resolved.
+
         """
 
         sys.path.append(os.getcwd())
@@ -82,7 +86,14 @@ def publish(self):
 
         logger.info("Starting to bootstrap")
         try:
-            graph = config.bootstrap()
+            try:
+                graph = config.bootstrap(strict_mode=strict_mode)
+                # TODO
+            except TypeError:
+                logger.warning(
+                    "Please pass kwargs into bootstrap! this will be deprecated soon!"
+                )
+                graph = config.bootstrap()
         except KeyboardInterrupt:
             return "Manual exit"
         except Exception as e:

src/codoc/service/graph.py

@@ -12,15 +12,19 @@
 
 
 def create_graph_of_module(
-    module: types.ModuleType, include_external_dependencies: bool = True
+    module: types.ModuleType,
+    include_external_dependencies: bool = True,
+    strict_mode: bool = True,
 ) -> Graph:
     all_objects = frozenset(recursively_get_all_subobjects_in_object(module))
     nodes = set(
         node
         for obj in all_objects
         for node in {create_node_from_object(obj)}
         | get_dependency_nodes_with_parents(
-            obj, include_external_dependencies=include_external_dependencies
+            obj,
+            include_external_dependencies=include_external_dependencies,
+            strict_mode=strict_mode,
         )
     )
     edges = set(
@@ -30,7 +34,9 @@ def create_graph_of_module(
         )
         for obj in all_objects
         for dependency in get_dependency_nodes(
-            obj, include_external_dependencies=include_external_dependencies
+            obj,
+            include_external_dependencies=include_external_dependencies,
+            strict_mode=strict_mode,
         )
     )
     return Graph(nodes=nodes, edges=edges)

src/codoc/service/parsing/dependency.py

@@ -23,6 +23,14 @@ class UnexpectedBuiltinError(Exception):
     ...
 
 
+class DependencyNotFound(Exception):
+    def __init__(self, obj, attr_name, module):
+        module_name = getattr(module, "__file__", str(module))
+        super().__init__(
+            f"Could not find `{attr_name}` in `{obj}` (in file {module_name})"
+        )
+
+
 def get_dependency_nodes(
     obj: ObjectType,
     **kwargs,
@@ -61,6 +69,7 @@ def bootstrap_kwargs(kwargs):
     kwargs.setdefault("create_node", create_node_from_object)
     kwargs.setdefault("get_parent", get_parent_of_object)
     kwargs.setdefault("include_external_dependencies", True)
+    kwargs.setdefault("strict_mode", True)
     return kwargs
 
 
@@ -75,6 +84,7 @@ def __init__(
         create_node: Callable[[ObjectType], Node],
         get_parent: Callable[[ObjectType], ObjectType],
         include_external_dependencies: bool,
+        strict_mode: bool,
     ):
         if is_builtin(obj):
             raise UnexpectedBuiltinError()
@@ -86,6 +96,7 @@ def __init__(
         self.create_node = create_node
         self.get_parent = get_parent
         self._include_external_dependencies = include_external_dependencies
+        self._strict_mode = strict_mode
         self._node = create_node(obj)
         self._module = inspect.getmodule(self._obj)
 
@@ -145,14 +156,17 @@ def get_object_from_module(self, identifier: str) -> Optional[ObjectType]:
         except AttributeError:
             pass
         try:
-            return _recursively_get_member_in_object_matching_identifier_name(
+            return self._recursively_get_member_in_object_matching_identifier_name(
                 identifier, self._module
             )
-        except AttributeError:
-            logger.error(
-                f"AttributeError when trying to fetch '{identifier}' when analyzing '{self._obj}'"
-            )
-            raise
+        except DependencyNotFound:
+            if self._strict_mode:
+                raise
+            else:
+                logger.warning(
+                    f"Could not find '{identifier}' in '{self._node.name}'({self._node.path})"
+                    "(skipping due to non-strict mode)"
+                )
 
     def get_referenced_identifier_names(self) -> Set[str]:
         try:
@@ -185,32 +199,31 @@ def recursively_get_parents(self, obj: ObjectType) -> Set[ObjectType]:
         else:
             return {parent} | self.recursively_get_parents(parent)
 
-
-def _recursively_get_member_in_object_matching_identifier_name(
-    identifier: str, current_object: ObjectType
-) -> Optional[ObjectType]:
-    split_identifier = identifier.split(".")
-    if len(split_identifier) == 1:
+    def _recursively_get_member_in_object_matching_identifier_name(
+        self, identifier: str, current_object: ObjectType
+    ) -> Optional[ObjectType]:
+        split_identifier = identifier.split(".")
+        if len(split_identifier) == 1:
+            try:
+                return getattr(current_object, identifier)
+            except AttributeError as e:
+                return handle_attribute_error_in_object_inspection(
+                    e, identifier, current_object, self._module
+                )
+        sub_identifier = ".".join(split_identifier[1:])
         try:
-            return getattr(current_object, identifier)
+            new_object = getattr(current_object, split_identifier[0])
+            return self._recursively_get_member_in_object_matching_identifier_name(
+                sub_identifier, new_object
+            )
         except AttributeError as e:
             return handle_attribute_error_in_object_inspection(
-                e, identifier, current_object
+                e, split_identifier[0], current_object, self._module
             )
-    sub_identifier = ".".join(split_identifier[1:])
-    try:
-        new_object = getattr(current_object, split_identifier[0])
-        return _recursively_get_member_in_object_matching_identifier_name(
-            sub_identifier, new_object
-        )
-    except AttributeError as e:
-        return handle_attribute_error_in_object_inspection(
-            e, split_identifier[0], current_object
-        )
 
 
 def handle_attribute_error_in_object_inspection(
-    error: Exception, identifier: str, current_object: ObjectType
+    error: Exception, identifier: str, current_object: ObjectType, module
 ) -> Optional[ObjectType]:
     # Handle special cases - i.e subprocess having OS specific parts
     if current_object in [subprocess, os]:
@@ -227,4 +240,5 @@ def handle_attribute_error_in_object_inspection(
             "Assuming that you are shadowing a builtin."
         )
         return None
-    raise error
+
+    raise DependencyNotFound(current_object, identifier, module) from error