Merge branch 'tickets/DM-26874' into master

bin.src/pipetask

@@ -22,5 +22,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import sys
-from lsst.ctrl.mpexec.cmdLineFwk import CmdLineFwk
-sys.exit(CmdLineFwk().parseAndRun())
+
+from lsst.ctrl.mpexec.cli.pipetask import main
+
+if __name__ == "__main__":
+    sys.exit(main())

python/lsst/ctrl/mpexec/__init__.py

@@ -20,7 +20,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from .cmdLineFwk import *
-from .cmdLineParser import *
 from .dotTools import *
 from .executionGraphFixup import *
 from .mpGraphExecutor import *

python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -25,6 +25,7 @@
                                      config_option,
                                      options_file_option)
 from lsst.daf.butler.cli.utils import (cli_handle_exception,
+                                       Mocker,
                                        MWCommand,
                                        MWCtxObj,
                                        option_section,
@@ -118,9 +119,21 @@ def qgraph(ctx, **kwargs):
 @ctrlMpExecOpts.meta_info_options()
 @option_section(sectionText="")
 @options_file_option()
+# --call-mocker is for use with test code, it is not intended for CLI or other
+# non-testing use. It allows this command function to be executed
+# programatically and have it call Mocker with its kwargs, which can the be
+# gotten from Mocker later. At some point, ctrl_mpexec should stop passing
+# around a SimpleNamespace of arguments, which would make this workaround
+# unnecessary.
+@click.option("--call-mocker",
+              is_flag=True,
+              hidden=True)  # do not show this option in the help menu.
 def run(ctx, **kwargs):
     """Build and execute pipeline and quantum graph.
     """
+    if kwargs["call_mocker"]:
+        Mocker(**kwargs)
+        return
     pipeline = _doBuild(ctx, **kwargs)
     qgraph = cli_handle_exception(script.qgraph, pipelineObj=pipeline, **kwargs)
     cli_handle_exception(script.run, qgraphObj=qgraph, **kwargs)

python/lsst/ctrl/mpexec/cli/opt/options.py

@@ -72,7 +72,7 @@
 # specified by the input_option callback defined in commands.py. Should
 # that callback definition get moved here? Should these defaults be made
 # less use-case specific and moved to commands.py? Is it ok as is?
-input_option = MWOptionDecorator("--input",
+input_option = MWOptionDecorator("-i", "--input",
                                  callback=split_commas,
                                  default=list(),
                                  help=unwrap("""Comma-separated names of the input collection(s). Entries may

python/lsst/ctrl/mpexec/cli/script/build.py

@@ -23,7 +23,7 @@
 
 from lsst.daf.butler.cli.cliLog import CliLog
 from ... import CmdLineFwk
-from ...cmdLineParser import _PipelineAction
+from ..utils import _PipelineAction
 
 
 def build(order_pipeline, pipeline, pipeline_actions, pipeline_dot, save_pipeline, show, log_level, **kwargs):

python/lsst/ctrl/mpexec/cli/utils.py

@@ -20,15 +20,70 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import collections
+import re
+
 from lsst.daf.butler.cli.opt import (config_file_option, config_option)
 from lsst.obs.base.cli.opt import instrument_option
 from .opt import (delete_option,
                   task_option)
-from ..cmdLineParser import (_ACTION_ADD_TASK,
-                             _ACTION_DELETE_TASK,
-                             _ACTION_CONFIG,
-                             _ACTION_CONFIG_FILE,
-                             _ACTION_ADD_INSTRUMENT)
+
+
+# Class which determines an action that needs to be performed
+# when building pipeline, its attributes are:
+#   action: the name of the action, e.g. "new_task", "delete_task"
+#   label:  task label, can be None if action does not require label
+#   value:  argument value excluding task label.
+_PipelineAction = collections.namedtuple("_PipelineAction", "action,label,value")
+
+
+class _PipelineActionType:
+    """Class defining a callable type which converts strings into
+    ``_PipelineAction`` instances.
+
+    Parameters
+    ----------
+    action : `str`
+        Name of the action, will become `action` attribute of instance.
+    regex : `str`
+        Regular expression for argument value, it can define groups 'label'
+        and 'value' which will become corresponding attributes of a
+        returned instance.
+    """
+
+    def __init__(self, action, regex='.*', valueType=str):
+        self.action = action
+        self.regex = re.compile(regex)
+        self.valueType = valueType
+
+    def __call__(self, value):
+        match = self.regex.match(value)
+        if not match:
+            raise TypeError("Unrecognized option syntax: " + value)
+        # get "label" group or use None as label
+        try:
+            label = match.group("label")
+        except IndexError:
+            label = None
+        # if "value" group is not defined use whole string
+        try:
+            value = match.group("value")
+        except IndexError:
+            pass
+        value = self.valueType(value)
+        return _PipelineAction(self.action, label, value)
+
+    def __repr__(self):
+        """String representation of this class.
+        """
+        return f"_PipelineActionType(action={self.action})"
+
+
+_ACTION_ADD_TASK = _PipelineActionType("new_task", "(?P<value>[^:]+)(:(?P<label>.+))?")
+_ACTION_DELETE_TASK = _PipelineActionType("delete_task", "(?P<value>)(?P<label>.+)")
+_ACTION_CONFIG = _PipelineActionType("config", "(?P<label>.+):(?P<value>.+=.+)")
+_ACTION_CONFIG_FILE = _PipelineActionType("configfile", "(?P<label>.+):(?P<value>.+)")
+_ACTION_ADD_INSTRUMENT = _PipelineActionType("add_instrument", "(?P<value>[^:]+)")
 
 
 def makePipelineActions(args,

python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -51,13 +51,11 @@
 import lsst.log
 import lsst.pex.config as pexConfig
 from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
-from .cmdLineParser import makeParser
 from .dotTools import graph2dot, pipeline2dot
 from .executionGraphFixup import ExecutionGraphFixup
 from .mpGraphExecutor import MPGraphExecutor
 from .preExecInit import PreExecInit
 from .singleQuantumExecutor import SingleQuantumExecutor
-from .taskFactory import TaskFactory
 from . import util
 from lsst.utils import doImport
 
@@ -443,66 +441,6 @@ class CmdLineFwk:
     def __init__(self):
         pass
 
-    def parseAndRun(self, argv=None):
-        """
-        This method is a main entry point for this class, it parses command
-        line and executes all commands.
-
-        Parameters
-        ----------
-        argv : `list` of `str`, optional
-            list of command line arguments, if not specified then
-            `sys.argv[1:]` is used
-        """
-
-        if argv is None:
-            argv = sys.argv[1:]
-
-        # start with parsing command line, only do partial parsing now as
-        # the tasks can add more arguments later
-        parser = makeParser()
-        args = parser.parse_args(argv)
-
-        # First thing to do is to setup logging.
-        self.configLog(args.longlog, args.loglevel)
-
-        taskFactory = TaskFactory()
-
-        # make pipeline out of command line arguments (can return empty pipeline)
-        try:
-            pipeline = self.makePipeline(args)
-        except Exception as exc:
-            print("Failed to build pipeline: {}".format(exc), file=sys.stderr)
-            raise
-
-        if args.subcommand == "build":
-            # stop here but process --show option first
-            self.showInfo(args, pipeline)
-            return 0
-
-        # make quantum graph
-        try:
-            qgraph = self.makeGraph(pipeline, args)
-        except Exception as exc:
-            print("Failed to build graph: {}".format(exc), file=sys.stderr)
-            raise
-
-        # optionally dump some info
-        self.showInfo(args, pipeline, qgraph)
-
-        if qgraph is None:
-            # No need to raise an exception here, code that makes graph
-            # should have printed warning message already.
-            return 2
-
-        if args.subcommand == "qgraph":
-            # stop here
-            return 0
-
-        # execute
-        if args.subcommand == "run":
-            return self.runPipeline(qgraph, taskFactory, args)
-
     @staticmethod
     def configLog(longlog, logLevels):
         """Configure logging system.