Merge branch 'tickets/DM-25385'

bin.src/pipetask2

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import sys
+
+from lsst.ctrl.mpexec.cli.pipetask import main
+
+if __name__ == "__main__":
+    sys.exit(main())

doc/lsst.ctrl.mpexec/index.rst

@@ -18,6 +18,20 @@ lsst.ctrl.mpexec
 .. .. toctree::
 ..    :maxdepth: 1
 
+.. _lsst.ctrl.pipetask-script:
+
+Command Line Scripts
+====================
+
+The `pipetask` command is being ported from an argparse framework to a Click
+framework. During development the command implemented using Click is called
+`pipetask2`. At some point the current `pipetask` command will be removed and
+`pipetask2` will be renamed to `pipetask`.
+
+.. click:: lsst.ctrl.mpexec.cli.pipetask:cli
+   :prog: pipetask2
+   :show-nested:
+
 .. _lsst.ctrl.mpexec-contributing:
 
 Contributing

python/lsst/ctrl/mpexec/cli/__init__.py

@@ -0,0 +1,20 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.

python/lsst/ctrl/mpexec/cli/cmd/__init__.py

@@ -0,0 +1,25 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ["build", "qgraph", "run"]
+
+
+from .commands import build, qgraph, run

python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -0,0 +1,103 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import click
+import copy
+
+from lsst.daf.butler.cli.opt import (config_file_option,
+                                     config_option,
+                                     log_level_option)
+from lsst.daf.butler.cli.utils import cli_handle_exception
+from lsst.obs.base.cli.opt import instrument_parameter
+from ..opt import (delete_option,
+                   order_pipeline_option,
+                   pipeline_dot_option,
+                   pipeline_option,
+                   save_pipeline_option,
+                   show_option,
+                   task_option)
+from .. import script
+from ..utils import makePipelineActions
+
+
+instrumentOptionHelp = ("Add an instrument which will be used to load config overrides when defining a "
+                        "pipeline. This must be the fully qualified class name.")
+
+
+class PipetaskCommand(click.Command):
+    def parse_args(self, ctx, args):
+        ctx.obj = copy.copy(args)
+        super().parse_args(ctx, args)
+
+
+@click.command(cls=PipetaskCommand, short_help="Build pipeline definition.")
+@click.pass_context
+@pipeline_option()
+@task_option(multiple=True)
+@delete_option(metavar="LABEL", multiple=True)
+@config_option(metavar="LABEL:NAME=VALUE", multiple=True)
+@config_file_option(help="Configuration override file(s), applies to a task with a given label.",
+                    metavar="LABEL:FILE",
+                    multiple=True)
+@order_pipeline_option()
+@save_pipeline_option()
+@pipeline_dot_option()
+@instrument_parameter(help=instrumentOptionHelp, metavar="instrument", multiple=True)
+@show_option(multiple=True)
+@log_level_option(defaultValue=None)
+def build(ctx, *args, **kwargs):
+    """Build and optionally save pipeline definition.
+
+    This does not require input data to be specified.
+    """
+    # The pipeline actions (task, delete, config, config_file, and instrument)
+    # must be handled in the order they appear on the command line, but the CLI
+    # specification gives them all different option names. So, instead of using
+    # the individual action options as they appear in kwargs (because
+    # invocation order can't be known), we capture the CLI arguments by
+    # overriding `click.Command.parse_args` and save them in the Context's
+    # `obj` parameter. We use `makePipelineActions` to create a list of
+    # pipeline actions from the CLI arguments and pass that list to the script
+    # function using the `pipeline_actions` kwarg name, and remove the action
+    # options from kwargs.
+    for pipelineAction in (task_option.optionKey, delete_option.optionKey, config_option.optionKey,
+                           config_file_option.optionKey, instrument_parameter.optionKey):
+        kwargs.pop(pipelineAction)
+    kwargs['pipeline_actions'] = makePipelineActions(ctx.obj)
+    cli_handle_exception(script.build, *args, **kwargs)
+
+
+@click.command(cls=PipetaskCommand)
+def qgraph(*args, **kwargs):
+    """Not implemented.
+
+    Build and optionally save pipeline and quantum graph.
+    """
+    print("Not implemented.")
+
+
+@click.command(cls=PipetaskCommand)
+def run(*args, **kwargs):
+    """Not implemented.
+
+    Build and execute pipeline and quantum graph.
+    """
+    print("Not implemented.")

python/lsst/ctrl/mpexec/cli/opt/__init__.py

@@ -0,0 +1,29 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from .delete import delete_option
+from .order_pipeline import order_pipeline_option
+from .pipeline import pipeline_option
+from .pipeline_dot import pipeline_dot_option
+from .save_pipeline import save_pipeline_option
+from .show import show_option
+from .task import task_option

python/lsst/ctrl/mpexec/cli/opt/delete.py

@@ -0,0 +1,47 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import click
+
+from lsst.daf.butler.core.utils import iterable
+from lsst.daf.butler.cli.utils import MWOption, split_commas
+
+
+class delete_option:  # noqa: N801
+
+    defaultHelp = "Delete task with given label from pipeline."
+    optionFlags = ("--delete")
+    optionKey = "delete"
+
+    def __init__(self, help=defaultHelp, metavar=None, multiple=False, required=False):
+        self.callback = split_commas if multiple else None
+        self.help = help
+        self.metavar = metavar
+        self.multiple = multiple
+        self.required = required
+
+    def __call__(self, f):
+        return click.option(*iterable(self.optionFlags), cls=MWOption,
+                            callback=self.callback,
+                            help=self.help,
+                            metavar=self.metavar,
+                            multiple=self.multiple,
+                            required=self.required)(f)

python/lsst/ctrl/mpexec/cli/opt/order_pipeline.py

@@ -0,0 +1,41 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import click
+
+from lsst.daf.butler.cli.utils import MWOption
+
+
+class order_pipeline_option:  # noqa: N801
+
+    defaultHelp = ("Order tasks in pipeline based on their data dependencies, ordering is performed as last "
+                   "step before saving or executing pipeline.")
+
+    def __init__(self, required=False, help=defaultHelp):
+        self.help = help
+        self.isFlag = True
+        self.required = required
+
+    def __call__(self, f):
+        return click.option("--order-pipeline", cls=MWOption,
+                            help=self.help,
+                            is_flag=self.isFlag,
+                            required=self.required)(f)

python/lsst/ctrl/mpexec/cli/opt/pipeline.py

@@ -0,0 +1,40 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import click
+
+from lsst.daf.butler.cli.utils import MWOption
+
+
+class pipeline_option:  # noqa: N801
+
+    defaultHelp = "Location of a pipeline definition file in YAML format."
+
+    def __init__(self, required=False, help=defaultHelp):
+        self.required = required
+        self.help = help
+        self.type = click.Path(exists=True, file_okay=True, dir_okay=False, readable=True)
+
+    def __call__(self, f):
+        return click.option("-p", "--pipeline", cls=MWOption,
+                            help=self.help,
+                            required=self.required,
+                            type=self.type)(f)