Skip to content

Commit

Permalink
Make command-line options order-independent (DM-21890)
Browse files Browse the repository at this point in the history 
This commit appears on DM-21889 branch even though the ticket for it is
DM-21890, this is to minimize the number of incompatible merges.

All global/top-level options have been removed and copied to each
sub-command (where they make sense). I have also refactored the code
that makes options into a number of separate methods, this makes it
cleaner and should also prepare us for the next step of refactoring. All
unit tests were updated for new command line interface.
  • Loading branch information
@andy-slac
andy-slac committed Nov 9, 2019
1 parent 2a97ddd commit c36ae7c
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 135 deletions.
280 changes: 167 additions & 113 deletions python/lsst/ctrl/mpexec/cmdLineParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""Module defining CmdLineParser class and related methods.
"""Module defining `makeParser` factory method.
"""

__all__ = ["makeParser"]
Expand Down Expand Up @@ -208,42 +208,14 @@ def _outputCollectionType(value):
* blank lines and lines starting with # are ignored
"""

# ------------------------
# Exported definitions --
# ------------------------


def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs):
"""Make instance of command line parser for `CmdLineFwk`.

Creates instance of parser populated with all options that are supported
by command line activator. There is no additional logic in this class,
all semantics is handled by the activator class.
def _makeButlerOptions(parser):
"""Add a set of options for data butler to a parser.
Parameters
----------
fromfile_prefix_chars : `str`, optional
Prefix for arguments to be used as options files (default: `@`)
parser_class : `type`, optional
Specifies the class of the argument parser, by default
`ArgumentParser` is used.
kwargs : extra keyword arguments
Passed directly to `parser_class` constructor
Returns
-------
instance of `parser_class`
parser : `argparse.ArgumentParser`
"""

parser = parser_class(usage="%(prog)s [global-options] subcommand [command-options]",
fromfile_prefix_chars=fromfile_prefix_chars,
epilog=_EPILOG,
formatter_class=RawDescriptionHelpFormatter,
**kwargs)

# global options which come before sub-command

# butler options
group = parser.add_argument_group("Data repository and selection options")
group.add_argument("-b", "--butler-config", dest="butler_config", default=None, metavar="PATH",
help="Location of the gen3 butler/registry config file.")
Expand All @@ -263,8 +235,24 @@ def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs)
group.add_argument("-d", "--data-query", dest="data_query", default="", metavar="QUERY",
help="User data selection expression.")

# output options

def _makeMetaOutputOptions(parser):
"""Add a set of options describing output metadata.
Parameters
----------
parser : `argparse.ArgumentParser`
"""
group = parser.add_argument_group("Meta-information output options")
group.add_argument("--skip-init-writes", dest="skip_init_writes", default=False,
action="store_true",
help="Do not write collection-wide 'init output' datasets (e.g. schemas).")
group.add_argument("--init-only", dest="init_only", default=False,
action="store_true",
help=("Do not actually run; just register dataset types and/or save init outputs."))
group.add_argument("--register-dataset-types", dest="register_dataset_types", default=False,
action="store_true",
help="Register DatasetTypes that do not already exist in the Registry.")
group.add_argument("--clobber-config", action="store_true", dest="clobberConfig", default=False,
help=("backup and then overwrite existing config files instead of checking them "
"(safe with -j, but not all other forms of parallel execution)"))
Expand All @@ -276,13 +264,114 @@ def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs)
group.add_argument("--no-versions", action="store_true", dest="noVersions", default=False,
help="don't check package versions; useful for development")

# logging/debug options
group = parser.add_argument_group("Execution and logging options")

def _makeLoggingOptions(parser):
"""Add a set of options for logging configuration.
Parameters
----------
parser : `argparse.ArgumentParser`
"""
group = parser.add_argument_group("Logging options")
group.add_argument("-L", "--loglevel", action=_LogLevelAction, default=[],
help="logging level; supported levels are [trace|debug|info|warn|error|fatal]",
metavar="LEVEL|COMPONENT=LEVEL")
group.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging")
group.add_argument("--debug", action="store_true", help="enable debugging output?")
group.add_argument("--debug", action="store_true", help="(deprecated) enable debugging output")


def _makePipelineOptions(parser):
"""Add a set of options for building a pipeline.
Parameters
----------
parser : `argparse.ArgumentParser`
"""
group = parser.add_argument_group("Pipeline building options")
group.add_argument("-p", "--pipeline", dest="pipeline",
help="Location of a serialized pipeline definition (pickle file).",
metavar="PATH")
group.add_argument("-t", "--task", metavar="TASK[:LABEL]",
dest="pipeline_actions", action='append', type=_ACTION_ADD_TASK,
help="Task name to add to pipeline, must be a fully qualified task name. "
"Task name can be followed by colon and "
"label name, if label is not given than task base name (class name) "
"is used as label.")
group.add_argument("--delete", metavar="LABEL",
dest="pipeline_actions", action='append', type=_ACTION_DELETE_TASK,
help="Delete task with given label from pipeline.")
group.add_argument("-c", "--config", metavar="LABEL:NAME=VALUE",
dest="pipeline_actions", action='append', type=_ACTION_CONFIG,
help="Configuration override(s) for a task with specified label, "
"e.g. -c task:foo=newfoo -c task:bar.baz=3.")
group.add_argument("-C", "--configfile", metavar="LABEL:PATH",
dest="pipeline_actions", action='append', type=_ACTION_CONFIG_FILE,
help="Configuration override file(s), applies to a task with a given label.")
group.add_argument("--order-pipeline", dest="order_pipeline",
default=False, action="store_true",
help="Order tasks in pipeline based on their data dependencies, "
"ordering is performed as last step before saving or executing "
"pipeline.")
group.add_argument("-s", "--save-pipeline", dest="save_pipeline",
help="Location for storing a serialized pipeline definition (pickle file).",
metavar="PATH")
group.add_argument("--pipeline-dot", dest="pipeline_dot",
help="Location for storing GraphViz DOT representation of a pipeline.",
metavar="PATH")
group.add_argument("--instrument", metavar="instrument",
dest="pipeline_actions", action="append", type=_ACTION_ADD_INSTRUMENT,
help="Add an instrument which will be used to load config overrides when"
" defining a pipeline. This must be the fully qualified class name")


def _makeQuntumGraphOptions(parser):
"""Add a set of options controlling quantum graph generation.
Parameters
----------
parser : `argparse.ArgumentParser`
"""
group = parser.add_argument_group("Quantum graph building options")
group.add_argument("-g", "--qgraph", dest="qgraph",
help="Location for a serialized quantum graph definition "
"(pickle file). If this option is given then all input data "
"options and pipeline-building options cannot be used.",
metavar="PATH")
groupex = group.add_mutually_exclusive_group()
groupex.add_argument("--skip-existing", dest="skip_existing",
default=False, action="store_true",
help="If all Quantum outputs already exist in output collection "
"then Quantum will be excluded from QuantumGraph.")
groupex.add_argument("--clobber-output", dest="clobber_output",
default=False, action="store_true",
help="Ignore or replace existing output datasets in output collecton. "
"With this option existing output datasets are ignored when generating "
"QuantumGraph, and they are removed from a collection prior to "
"executing individual Quanta. This option is exclusive with "
"--skip-existing option.")
group.add_argument("-q", "--save-qgraph", dest="save_qgraph",
help="Location for storing a serialized quantum graph definition "
"(pickle file).",
metavar="PATH")
group.add_argument("--save-single-quanta", dest="save_single_quanta",
help="Format string of locations for storing individual quantum graph "
"definition (pickle files). The curly brace {} in the input string "
"will be replaced by a quantum number.",
metavar="PATH")
group.add_argument("--qgraph-dot", dest="qgraph_dot",
help="Location for storing GraphViz DOT representation of a "
"quantum graph.",
metavar="PATH")


def _makeExecOptions(parser):
"""Add options controlling how tasks are executed.
Parameters
----------
parser : `argparse.ArgumentParser`
"""
group = parser.add_argument_group("Execution options")
group.add_argument("--doraise", action="store_true",
help="raise an exception on error (else log a message and continue)?")
group.add_argument("--profile", metavar="PATH", help="Dump cProfile statistics to filename")
Expand All @@ -292,6 +381,39 @@ def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs)
group.add_argument("--timeout", type=float,
help="Timeout for multiprocessing; maximum wall time (sec)")

# ------------------------
# Exported definitions --
# ------------------------


def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs):
"""Make instance of command line parser for `CmdLineFwk`.
Creates instance of parser populated with all options that are supported
by command line activator. There is no additional logic in this class,
all semantics is handled by the activator class.
Parameters
----------
fromfile_prefix_chars : `str`, optional
Prefix for arguments to be used as options files (default: `@`)
parser_class : `type`, optional
Specifies the class of the argument parser, by default
`ArgumentParser` is used.
kwargs : extra keyword arguments
Passed directly to `parser_class` constructor
Returns
-------
instance of `parser_class`
"""

parser = parser_class(usage="%(prog)s subcommand [options]",
fromfile_prefix_chars=fromfile_prefix_chars,
epilog=_EPILOG,
formatter_class=RawDescriptionHelpFormatter,
**kwargs)

# define sub-commands
subparsers = parser.add_subparsers(dest="subcommand",
title="commands",
Expand Down Expand Up @@ -323,85 +445,17 @@ def makeParser(fromfile_prefix_chars='@', parser_class=ArgumentParser, **kwargs)
formatter_class=RawDescriptionHelpFormatter)
subparser.set_defaults(subparser=subparser,
pipeline_actions=[])
_makeLoggingOptions(subparser)
_makePipelineOptions(subparser)

if subcommand in ("qgraph", "run"):
subparser.add_argument("-g", "--qgraph", dest="qgraph",
help="Location for a serialized quantum graph definition "
"(pickle file). If this option is given then all input data "
"options and pipeline-building options cannot be used.",
metavar="PATH")
_makeQuntumGraphOptions(subparser)
_makeButlerOptions(subparser)

if subcommand == "run":
subparser.add_argument("--skip-init-writes", dest="skip_init_writes", default=False,
action="store_true",
help="Do not write collection-wide 'init output' datasets (e.g. schemas).")
subparser.add_argument("--init-only", dest="init_only", default=False,
action="store_true",
help=("Do not actually run; just register dataset types and/or save "
"init outputs."))
subparser.add_argument("--register-dataset-types", dest="register_dataset_types", default=False,
action="store_true",
help="Register DatasetTypes that do not already exist in the Registry.")
subparser.add_argument("-p", "--pipeline", dest="pipeline",
help="Location of a serialized pipeline definition (pickle file).",
metavar="PATH")
subparser.add_argument("-t", "--task", metavar="TASK[:LABEL]",
dest="pipeline_actions", action='append', type=_ACTION_ADD_TASK,
help="Task name to add to pipeline, must be a fully qualified task name. "
"Task name can be followed by colon and "
"label name, if label is not given than task base name (class name) "
"is used as label.")
subparser.add_argument("--delete", metavar="LABEL",
dest="pipeline_actions", action='append', type=_ACTION_DELETE_TASK,
help="Delete task with given label from pipeline.")
subparser.add_argument("-c", "--config", metavar="LABEL:NAME=VALUE",
dest="pipeline_actions", action='append', type=_ACTION_CONFIG,
help="Configuration override(s) for a task with specified label, "
"e.g. -c task:foo=newfoo -c task:bar.baz=3.")
subparser.add_argument("-C", "--configfile", metavar="LABEL:PATH",
dest="pipeline_actions", action='append', type=_ACTION_CONFIG_FILE,
help="Configuration override file(s), applies to a task with a given label.")
subparser.add_argument("--order-pipeline", dest="order_pipeline",
default=False, action="store_true",
help="Order tasks in pipeline based on their data dependencies, "
"ordering is performed as last step before saving or executing "
"pipeline.")
subparser.add_argument("--instrument", metavar="instrument",
dest="pipeline_actions", action="append", type=_ACTION_ADD_INSTRUMENT,
help="Add an instrument which will be used to load config overrides when"
"defining a pipeline. This must be the fully qualified class name")
if subcommand in ("qgraph", "run"):
group = subparser.add_mutually_exclusive_group()
group.add_argument("--skip-existing", dest="skip_existing",
default=False, action="store_true",
help="If all Quantum outputs already exist in output collection "
"then Quantum will be excluded from QuantumGraph.")
group.add_argument("--clobber-output", dest="clobber_output",
default=False, action="store_true",
help="Ignore or replace existing output datasets in output collecton. "
"With this option existing output datasets are ignored when generating "
"QuantumGraph, and they are removed from a collection prior to "
"executing individual Quanta. This option is exclusive with "
"--skip-existing option.")
subparser.add_argument("-s", "--save-pipeline", dest="save_pipeline",
help="Location for storing a serialized pipeline definition (pickle file).",
metavar="PATH")
if subcommand in ("qgraph", "run"):
subparser.add_argument("-q", "--save-qgraph", dest="save_qgraph",
help="Location for storing a serialized quantum graph definition "
"(pickle file).",
metavar="PATH")
subparser.add_argument("--save-single-quanta", dest="save_single_quanta",
help="Format string of locations for storing individual quantum graph "
"definition (pickle files). The curly brace {} in the input string "
"will be replaced by a quantum number.",
metavar="PATH")
subparser.add_argument("--pipeline-dot", dest="pipeline_dot",
help="Location for storing GraphViz DOT representation of a pipeline.",
metavar="PATH")
if subcommand in ("qgraph", "run"):
subparser.add_argument("--qgraph-dot", dest="qgraph_dot",
help="Location for storing GraphViz DOT representation of a "
"quantum graph.",
metavar="PATH")
_makeExecOptions(subparser)
_makeMetaOutputOptions(subparser)

subparser.add_argument("--show", metavar="ITEM|ITEM=VALUE", action="append", default=[],
help="Dump various info to standard output. Possible items are: "
"`config', `config=[Task/]<PATTERN>' or "
Expand Down

0 comments on commit c36ae7c

Please sign in to comment.