Merge pull request #74 from vogt4nick/2019-03-03-misc-dev

Create `commons` submodule, test, and document

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.5
+current_version = 0.9.0
 commit = True
 tag = False
 parse = (?P<major>\d+)

dequindre/__init__.py

@@ -14,12 +14,10 @@
 from subprocess import run as subprocess_run
 from subprocess import check_output, CalledProcessError
 
+from dequindre.exceptions import CyclicGraphError
 
-__version__ = '0.8.5'
 
-
-class CyclicGraphError(Exception):
-    pass
+__version__ = '0.9.0'
 
 
 class Task:

dequindre/commons.py

@@ -0,0 +1,99 @@
+"""Environment path shortcuts for easier access.
+
+The three main virtual environment modules follow different structures. We 
+define three functions to simplify paths to these virtual environments.
+
+- venv_shortcut
+- pipenv_shortcut
+- conda_shortcut
+"""
+
+from contextlib import contextmanager
+from os.path import join as pathjoin
+from typing import Callable
+
+from dequindre import Task
+
+
+@contextmanager
+def common_task(common_loc: str, common_env: str = 'python'):
+    """Create tasks with a common parent path and environment.
+    
+    Lots of tasks will use the same environment or same directory. These 
+    `commons` reduce duplicate code.
+
+    Args:
+        common_loc (str): {}-formatted parent path.
+        common_env (str, optional): environment.
+    """
+    assert isinstance(common_loc, str), '`common_loc` must be a str'
+    assert len(common_loc) > 0, '`common_loc` must not be an empty str'
+    assert '{' in common_loc and '}' in common_loc
+    assert isinstance(common_env, str)
+    assert len(common_env) > 0, '`common_env` must not be an empty str'
+
+    def construct_task(loc: str):
+        return Task(common_loc.format(loc), env=common_env)
+
+    yield construct_task
+
+
+@contextmanager
+def common_venv(common_prefix: str = '.', 
+                common_suffix: str = None) \
+                -> Callable:
+    """Quickly construct a path to a common virtualenv environment
+
+    venv follows the structure: `/path/to/{{my_env}}/Scripts/python`
+
+    Args:
+        common_prefix (str): The file path before the environment name.
+        common_suffix (str, optional): The file path after the environment name.
+
+    Returns:
+        Function to shorten env specification
+    """
+    if common_suffix is None:
+        common_suffix = pathjoin('Scripts', 'python')
+    yield lambda s: pathjoin(common_prefix, s, common_suffix)
+
+
+@contextmanager
+def common_pipenv(common_prefix: str = '.', 
+                  common_suffix: str = None) \
+                  -> Callable:
+    """Quickly construct a path to a common pipenv environment
+
+    pipenv follows the structure: `/path/to/{{my_env}}/Scripts/python`
+
+    Args:
+        common_prefix (str): The file path before the environment name.
+        common_suffix (str, optional): The file path after the environment name.
+
+    Returns:
+        Function to shorten env specification
+    """
+    if common_suffix is None:
+        common_suffix = pathjoin('Scripts', 'python')
+    yield lambda s: pathjoin(common_prefix, s, common_suffix)
+
+
+@contextmanager
+def common_conda_env(common_prefix: str, 
+                     common_suffix: str = None) \
+                     -> Callable:
+    """Quickly construct a path to a common conda environment
+
+    conda follows the structure: `/path/to/conda/envs/{{my_env}}/bin/python`
+
+    Args:
+        common_prefix (str): The file path before the environment name.
+        common_suffix (str, optional): The file path after the environment name.
+
+    Returns:
+        Function to shorten env specification
+    """
+    if common_suffix is None:
+        common_suffix = pathjoin('bin', 'python')
+    yield lambda s: pathjoin(common_prefix, s, common_suffix)
+

dequindre/exceptions.py

@@ -0,0 +1,3 @@
+
+class CyclicGraphError(Exception):
+    pass

docs/source/api-docs/api-docs.rst

@@ -0,0 +1,9 @@
+API Documentation
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   dequindre-module
+   dequindre-commons-module
+   dequindre-exceptions-module

docs/source/api-docs/dequindre-commons-module.rst

@@ -0,0 +1,7 @@
+Commons Submodule
+=================
+
+.. automodule:: dequindre.commons
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/api-docs/dequindre-exceptions-module.rst

@@ -0,0 +1,7 @@
+Exceptions Submodule
+====================
+
+.. automodule:: dequindre.exceptions
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/autodoc/dequindre-module.rst → docs/source/api-docs/dequindre-module.rst

@@ -1,9 +1,6 @@
 Dequindre Module
 ================
 
-Module contents
----------------
-
 .. automodule:: dequindre
     :members:
     :undoc-members:

docs/source/conf.py

@@ -16,7 +16,7 @@
 import sys
 sys.path.insert(0, os.path.abspath('.'))
 sys.path.insert(0, os.path.abspath('../'))
-
+sys.path.insert(0, os.path.abspath('../..'))
 
 # -- Project information -----------------------------------------------------
 
@@ -27,7 +27,7 @@
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '0.8.5'
+release = '0.9.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -7,7 +7,7 @@ User Guide
 .. toctree::
    :maxdepth: 2
 
-   user-guide/intro
-   user-guide/design
+   api-docs/api-docs
    user-guide/cookbook
-   autodoc/dequindre-module
+   user-guide/design
+   user-guide/philosophy

docs/source/user-guide/cookbook.rst

@@ -2,200 +2,12 @@
 Cookbook
 ========
 
-This cookbook will make use of three sample python files.
+.. include:: ./cookbook/example-files.rst
 
-.. code-block:: python
+.. include:: ./cookbook/tasks.rst
 
-    ## ./boil_water.py
-    print("I am boiling water...")
+.. include:: ./cookbook/commons.rst
 
-.. code-block:: python
+.. include:: ./cookbook/dags.rst
 
-    ## ./steep_tea.py
-    print("I am steeping tea...")
-
-.. code-block:: python
-
-    ## ./pour_tea.py
-    print("I am pouring tea...")
-
-We also use Git Bash as the terminal. Bash commands work on windows and unix 
-machines unless otherwise stated.
-
-Tasks
------
-
-Configure a Task
-~~~~~~~~~~~~~~~~
-
-.. code-block:: python
-
-    >>> from dequindre import Task
-    >>> pour_tea = Task('./pour_tea.py')
-    >>> pour_tea
-    Task(./pour_tea.py)
-    >>> pour_tea.loc
-    './pour_tea.py'
-    >>> pour_tea.env
-    'python'
-
-Note that that the python environment defaulted to 'python'. To use different 
-environments, we'll need to define them first.
-
-virtualenv Environments
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Suppose you want to run tasks in a different virtualenv environment. Let's 
-define a virtualenv environment:
-
-.. code-block:: bash
-
-    $ virtualenv venv
-    ...
-
-virtualenv envirionments have a defined structure. The path to the python 
-executable is `./venv/Scripts/python`. This will become our task env.
-
-.. code-block:: python
-
-    >>> from dequindre import Task
-    >>> venv = './venv/Scripts/python'
-    >>> pour_tea = Task('./pour_tea.py', env=venv)
-    >>> pour_tea.env
-    './venv/Scripts/python'
-
-Now the task will run in the specified environment at runtime.
-
-### Create Tasks in pipenv Environments
-
-pipenv environments follow the same structure as virtualenv environments. They
-may be be located elsewhere on you file system. Finding it is easy. Note that 
-you may have to delete your recently created `venv` directory.
-
-.. code-block:: bash
-
-    $ pipenv install dequindre
-    ...
-    $ pipenv shell
-    Launching subshell in virtual environment
-    $ where python
-    /your/path/to/.virtualenvs/dequindre-5srOTnbr/Scripts/python
-
-The output will be different on your machine, and there may be multiple paths,
-but the pipenv path will include the `.virtualenvs/` directory.
-
-.. code-block:: python
-
-    >>> from dequindre import Task
-    >>> from os.path import join as pathjoin
-    >>> PIPENV_DIR = '/path/to/your/.virtualenvs'
-
-    >>> dequindre_env = pathjoin(PIPENV_DIR, 'dequindre-5srOTnbr', 
-                                 'Scripts', 'python')
-    >>> pour_tea = Task('./pour_tea.py', env=dequindre_env)
-    >>> pour_tea.env
-    '/your/path/to/.virtualenvs/dequindre-5srOTnbr/Scripts/python'
-
-Now the task is pointing to the pipenv environment and will run that 
-environment at runtime.
-
-conda Environments
-~~~~~~~~~~~~~~~~~~
-
-Suppose you want to run tasks using your conda environments. Conda 
-environments are slightly trickier than virtualenv environments.
-
-First, create a test environment and find where your conda installation is 
-located. You'll ought to see something like
-
-.. code-block:: bash
-
-    $ conda create -n test_env python=3.6
-    ...
-    $ where conda
-    /your/path/to/miniconda3/condabin/
-    /your/path/to/miniconda3/Scripts/conda
-    /your/path/to/miniconda3/Library/bin/conda
-
-The output will be different on your machine, but the important directory is 
-the common directory; in this case, it's miniconda3.
-
-conda, like virtualenv and pipenv, also has a well defined structure for 
-environments that looks like `miniconda3/envs/test_env/bin/python`.
-
-.. code-block:: python
-
-    >>> from dequindre import Task
-    >>> from os.path import join as pathjoin
-    >>> CONDA_DIR = '/your/path/to/miniconda3'
-
-    >>> test_env = pathjoin(CONDA_DIR, 'envs', 'test_env', 'bin', 'python')
-    >>> pour_tea = Task('./pour_tea.py', env=venv)
-    >>> pour_tea.env
-    '/your/path/to/miniconda3/envs/test_env/Scripts/python'
-
-Now the task is pointing to the conda environment and will run that environment at runtime.
-
-DAGs
-----
-
-Configure a DAG
-~~~~~~~~~~~~~~~
-
-.. code-block:: python
-
-    >>> from dequindre import Task, DAG
-
-    >>> ## define tasks
-    >>> boil_water = Task('./boil_water.py')
-    >>> steep_tea = Task('./steep_tea.py')
-    >>> pour_tea = Task('./pour_tea.py')
-
-    >>> make_tea = DAG()
-    >>> make_tea.add_dependencies({
-    ...       steep_tea: boil_water,
-    ...       pour_tea: steep_tea
-    ...   })
-
-
-Dequindre Schedulers
---------------------
-
-The Dequindre scheduler is the last major object in dequindre. After defining 
-your tasks and task dependencies in the DAG, you can create a Dequindre 
-scheduler.  
-
-.. code-block:: python
-
-    >>> from dequindre import Task, DAG, Dequindre
-
-    >>> ## define tasks
-    >>> boil_water = Task('./boil_water.py')
-    >>> steep_tea = Task('./steep_tea.py')
-    >>> pour_tea = Task('./pour_tea.py')
-
-    >>> make_tea = DAG()
-    >>> make_tea.add_dependencies({
-    ...       steep_tea: boil_water,
-    ...       pour_tea: steep_tea
-    ...   })
-
-    >>> dq = Dequindre(make_tea)
-    >>> dq.get_schedules()
-    defaultdict(<class 'set'>, {
-        1: {Task(boil_water.py)},  
-        2: {Task(steep_tea.py)},  
-        3: {Task(pour_tea.py)}})
-    >>> dq.run_tasks()
-
-    Running Task(./boil_water.py)
-
-    I am boiling water...
-
-    Running Task(./steep_tea.py)
-
-    I am steeping tea...
-
-    Running Task(./pour_tea.py)
-
-    I am pouring tea...
+.. include:: ./cookbook/dequindre-schedulers.rst