Merge 0254e39 into ce50ff2

CHANGES

@@ -22,7 +22,8 @@ Changes
  - Fix `result_dep`, use result **after** its execution
  - Fix #286: Support `functools.partial` on tasks' dict metadata `task.title`
  - Fix #285: `clean` command, remove targets in reverse lexical order
-
+ - Deprecated `TaskLoader` in favor of `TaskLoader2`, which separates loading doit's configuration from loading tasks.
+ - Added `doit.get_dep_manager()` to access dependency manager during all task processing phases.
 
 0.31.1 (*2018-03-18*)
 =====================

doc/extending.rst

@@ -31,11 +31,13 @@ Normally `doit` tasks are defined in a `dodo.py` file.
 This file is loaded, and the list of tasks is created from
 the dict containing task meta-data from the *task-creator* functions.
 
-Subclass TaskLoader to create a custom loader:
+Subclass ``TaskLoader2`` to create a custom loader:
 
-.. autoclass:: doit.cmd_base.TaskLoader
-   :members: load_tasks
+.. autoclass:: doit.cmd_base.TaskLoader2
+   :members:
 
+Before the introduction of ``TaskLoader2`` a now deprecated loader interface ``TaskLoader`` was
+used, which did not separate the setup, configuration loading and task loading phases explicit.
 
 The main program is implemented in the `DoitMain`. It's constructor
 takes an instance of the task loader to be used.

doc/samples/custom_loader.py

@@ -3,21 +3,26 @@
 import sys
 
 from doit.task import dict_to_task
-from doit.cmd_base import TaskLoader
+from doit.cmd_base import TaskLoader2
 from doit.doit_cmd import DoitMain
 
 my_builtin_task = {
     'name': 'sample_task',
     'actions': ['echo hello from built in'],
     'doc': 'sample doc',
-    }
+}
 
-class MyLoader(TaskLoader):
-    @staticmethod
-    def load_tasks(cmd, opt_values, pos_args):
+
+class MyLoader(TaskLoader2):
+    def setup(self, opt_values):
+        pass
+
+    def load_doit_config(self):
+        return {'verbosity': 2}
+
+    def load_tasks(self, cmd, pos_args):
         task_list = [dict_to_task(my_builtin_task)]
-        config = {'verbosity': 2}
-        return task_list, config
+        return task_list
 
 
 if __name__ == "__main__":

doit/__init__.py

@@ -33,11 +33,19 @@
 from doit.doit_cmd import get_var
 from doit.api import run
 from doit.tools import load_ipython_extension
+from doit.globals import Globals
 
 __all__ = ['get_var', 'run', 'create_after']
 
+
 def get_initial_workdir():
     """working-directory from where the doit command was invoked on shell"""
     return loader.initial_workdir
 
+
+def get_dep_manager():
+    """Dependency manager singleton."""
+    return Globals.dep_manager
+
+
 assert load_ipython_extension  # silence pyflakes

doit/cmd_base.py

@@ -4,6 +4,7 @@
 from collections import defaultdict
 import textwrap
 
+from .globals import Globals
 from . import version
 from .cmdparse import CmdOption, CmdParse
 from .exceptions import InvalidCommand, InvalidDodoFile
@@ -273,10 +274,8 @@ def help(self):
 }
 
 
-class TaskLoader(object):
-    """task-loader interface responsible of creating Task objects
-
-    Subclasses must implement the method `load_tasks`
+class TaskLoaderBase:
+    """Common attributes of task loaders.
 
     :cvar cmd_options:
           (list of dict) see cmdparse.CmdOption for dict format
@@ -286,8 +285,15 @@ class TaskLoader(object):
     def __init__(self):
         # list of command names, used to detect clash of task names and commands
         self.cmd_names = []
-        self.config = None # reference to config object taken from Command
+        self.config = None  # reference to config object taken from Command
+
 
+class TaskLoader(TaskLoaderBase):
+    """task-loader interface responsible of creating Task objects
+
+    Subclasses must implement the method `load_tasks`. Note: This interface is deprecated, use
+    `TaskLoader2` instead.
+    """
     def load_tasks(self, cmd, opt_values, pos_args): # pragma: no cover
         """load tasks and DOIT_CONFIG
 
@@ -310,30 +316,86 @@ def _load_from(cmd, namespace, cmd_list):
         return task_list, doit_config
 
 
-class ModuleTaskLoader(TaskLoader):
+class TaskLoader2(TaskLoaderBase):
+    """Interface of task loaders with new-style API.
+
+    The default implementation assumes tasks are loaded from a namespace, mapping identifiers to
+    elements like functions (like task generators) or constants (like configuration values).
+
+    This API update separates the loading of the configuration and the loading of the actual tasks,
+    which enables additional elements to be available during task creation.
+    """
+    API = 2
+
+    def setup(self, opt_values):
+        """Delayed initialization.
+
+        To be implemented if the data is needed by derived classes.
+
+        :param opt_values: (dict) with values for cmd_options
+        """
+
+    def load_doit_config(self):
+        """Load doit configuration.
+
+        The method must not be called before invocation of ``setup``.
+
+        :return: (dict) Dictionary of doit configuration values.
+        """
+        raise NotImplementedError()  # pragma: no cover
+
+    def load_tasks(self, cmd, pos_args):
+        """Load tasks.
+
+        The method must not be called before invocation of ``load_doit_config``.
+
+        :param cmd: (doit.cmd_base.Command) current command being executed
+        :param pos_args: (list str) positional arguments from command line
+        :return: (List[Task])
+        """
+        raise NotImplementedError()  # pragma: no cover
+
+
+class NamespaceTaskLoader(TaskLoader2):
+    """Implementation of a loader of tasks from an abstract namespace.
+
+    A namespace is simply a dictionary to objects like functions and objects. See the derived
+    classes for some concrete namespace types.
+    """
+    def __init__(self):
+        super().__init__()
+        self.namespace = None
+
+    def load_doit_config(self):
+        return loader.load_doit_config(self.namespace)
+
+    def load_tasks(self, cmd, pos_args):
+        return loader.load_tasks(self.namespace, self.cmd_names, cmd.execute_tasks)
+
+
+class ModuleTaskLoader(NamespaceTaskLoader):
     """load tasks from a module/dictionary containing task generators
     Usage: `ModuleTaskLoader(my_module)` or `ModuleTaskLoader(globals())`
     """
-    cmd_options = ()
-
     def __init__(self, mod_dict):
-        super(ModuleTaskLoader, self).__init__()
-        self.mod_dict = mod_dict
-
-    def load_tasks(self, cmd, params, args):
-        return self._load_from(cmd, self.mod_dict, self.cmd_names)
+        super().__init__()
+        if inspect.ismodule(mod_dict):
+            self.namespace = dict(inspect.getmembers(mod_dict))
+        else:
+            self.namespace = mod_dict
 
 
-class DodoTaskLoader(TaskLoader):
+class DodoTaskLoader(NamespaceTaskLoader):
     """default task-loader create tasks from a dodo.py file"""
     cmd_options = (opt_dodo, opt_cwd, opt_seek_file)
 
-    def load_tasks(self, cmd, params, args):
-        dodo_module = loader.get_module(
-            params['dodoFile'],
-            params['cwdPath'],
-            params['seek_file'])
-        return self._load_from(cmd, dodo_module, self.cmd_names)
+    def setup(self, opt_values):
+        # lazily load namespace from dodo file per config parameters:
+        self.namespace = dict(inspect.getmembers(loader.get_module(
+            opt_values['dodoFile'],
+            opt_values['cwdPath'],
+            opt_values['seek_file'],
+        )))
 
 
 def get_loader(config, task_loader=None, cmds=None):
@@ -446,8 +508,16 @@ def execute(self, params, args):
         :param params: instance of cmdparse.DefaultUpdate
         :param args: list of string arguments (containing task names)
         """
-        self.task_list, dodo_config = self.loader.load_tasks(
-            self, params, args)
+
+        # distinguish legacy and new-style task loader API when loading tasks:
+        legacy_loader = getattr(self.loader, 'API', 1) < 2
+        if legacy_loader:
+            self.task_list, dodo_config = self.loader.load_tasks(
+                self, params, args)
+        else:
+            self.loader.setup(params)
+            dodo_config = self.loader.load_doit_config()
+
         # merge config values from dodo.py into params
         params.update_defaults(dodo_config)
 
@@ -470,6 +540,13 @@ def execute(self, params, args):
             self.dep_manager = Dependency(
                 db_class, params['dep_file'], checker_cls)
 
+        # register dependency manager in global registry:
+        Globals.dep_manager = self.dep_manager
+
+        # load tasks from new-style loader, now that dependency manager is available:
+        if not legacy_loader:
+            self.task_list = self.loader.load_tasks(cmd=self, pos_args=args)
+
         # hack to pass parameter into _execute() calls that are not part
         # of command line options
         params['pos_args'] = args

doit/cmd_completion.py

@@ -6,7 +6,6 @@
 from .exceptions import InvalidCommand
 from .cmd_base import DoitCmdBase
 
-
 opt_shell = {
     'name': 'shell',
     'short': 's',
@@ -98,8 +97,13 @@ def _generate_bash(self, opt_values, pos_args):
 
         # if hardcode tasks
         if opt_values['hardcode_tasks']:
-            self.task_list, _ = self.loader.load_tasks(
-                self, opt_values, pos_args)
+            if getattr(self.loader, 'API', 1) == 2:
+                self.loader.setup(opt_values)
+                self.loader.load_doit_config()
+                self.task_list = self.loader.load_tasks(cmd=self, pos_args=pos_args)
+            else:
+                self.task_list, _ = self.loader.load_tasks(
+                    self, opt_values, pos_args)
             task_names = (t.name for t in self.task_list if not t.subtask_of)
             tmpl_vars['pt_tasks'] = '"{0}"'.format(' '.join(sorted(task_names)))
         else:
@@ -190,8 +194,13 @@ def _generate_zsh(self, opt_values, pos_args):
         }
 
         if opt_values['hardcode_tasks']:
-            self.task_list, _ = self.loader.load_tasks(
-                self, opt_values, pos_args)
+            if getattr(self.loader, 'API', 1) == 2:
+                self.loader.setup(opt_values)
+                self.loader.load_doit_config()
+                self.task_list = self.loader.load_tasks(cmd=self, pos_args=pos_args)
+            else:
+                self.task_list, _ = self.loader.load_tasks(
+                    self, opt_values, pos_args)
             lines = []
             for task in self.task_list:
                 if not task.subtask_of:

doit/globals.py

@@ -0,0 +1,9 @@
+"""Simple registry of singletons."""
+
+
+class Globals:
+    """Accessors to doit singletons.
+
+    :cvar dep_manager: The doit dependency manager, holding all persistent task data.
+    """
+    dep_manager = None

tests/module_with_tasks.py

@@ -0,0 +1,14 @@
+"""ModuleLoadTest uses this file to load tasks from module."""
+
+DOIT_CONFIG = dict(verbose=2)
+
+
+def task_xxx1():
+    return dict(actions=[])
+
+
+task_no = 'strings are not tasks'
+
+
+def blabla():
+    ...  # pragma: no cover

tests/test___init__.py

@@ -1,6 +1,8 @@
 import os
+from unittest import mock
 
 import doit
+from doit.globals import Globals
 from doit.loader import get_module
 
 
@@ -13,3 +15,7 @@ def test_get_initial_workdir(restore_cwd):
     assert os.getcwd() == cwd, os.getcwd()
     assert doit.get_initial_workdir() == initial_wd
 
+
+def test_get_dep_manager():
+    Globals.dep_manager = mock.sentinel.DEP_MANAGER
+    assert doit.get_dep_manager() == mock.sentinel.DEP_MANAGER