Add pipetask report on output of a quantum graph

doc/changes/DM-41131.feature.md

@@ -0,0 +1,4 @@
+Add `pipetask report` which reads a quantum graph and reports on the outputs
+of failed, produced and missing quanta. This is a command-line incarnation of
+`QuantumGraphExecutionReport.make_reports` in combination with
+`QuantumGraphExecutionReport.write_summary_yaml`.

python/lsst/ctrl/mpexec/cli/cmd/__init__.py

@@ -25,7 +25,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ["build", "cleanup", "pre_exec_init_qbb", "purge", "qgraph", "run", "run_qbb", "update_graph_run"]
+__all__ = ["build", "cleanup", "pre_exec_init_qbb", "purge", "qgraph", "report", "run", "run_qbb", "update_graph_run"]
 
 
-from .commands import build, cleanup, pre_exec_init_qbb, purge, qgraph, run, run_qbb, update_graph_run
+from .commands import build, cleanup, pre_exec_init_qbb, purge, qgraph, report, run, run_qbb, update_graph_run

python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -324,3 +324,19 @@ def update_graph_run(
     OUTPUT_QGRAPH is the URL to store the updated Quantum Graph.
     """
     script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)
+
+
+@click.command(cls=PipetaskCommand)
+@ctrlMpExecOpts.qgraph_argument()
+@ctrlMpExecOpts.butler_config_option()
+@click.argument("output_yaml", type=click.Path(exists=False))
+@click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.")
+def report(qgraph: str, butler_config: str, output_yaml: str, logs: bool = True) -> None:
+    """Write a yaml file summarizing the produced and missing expected datasets
+    in a quantum graph.
+
+    QGRAPH is the URL to a serialized Quantum Graph file.
+
+    OUTPUT_YAML is the URL to store the summary report.
+    """
+    script.report(qgraph, butler_config, output_yaml, logs)

python/lsst/ctrl/mpexec/cli/script/__init__.py

@@ -31,6 +31,7 @@
 from .pre_exec_init_qbb import pre_exec_init_qbb
 from .purge import PurgeResult, purge
 from .qgraph import qgraph
+from .report import report
 from .run import run
 from .run_qbb import run_qbb
 from .update_graph_run import update_graph_run

python/lsst/ctrl/mpexec/cli/script/report.py

@@ -0,0 +1,57 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from lsst.daf.butler import Butler
+from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.execution_reports import QuantumGraphExecutionReport
+
+
+def report(qgraph_uri: str, butler_config: str, output_yaml: str, logs: bool = True) -> None:
+    """Write a yaml file summarizing the produced and missing expected datasets
+    in a quantum graph.
+
+    Parameters
+    ----------
+        butler_config : `str`
+            The Butler used for this report. This should match the Butler used
+            for the run associated with the executed quantum graph.
+        qgraph_uri : `str`
+            The uri of the location of said quantum graph.
+        output_yaml : `str`
+            The name to be used for the summary yaml file.
+        logs : `bool`
+            Get butler log datasets for extra information.
+
+    See Also
+    --------
+        lsst.pipe.base.QuantumGraphExecutionReport.make_reports
+        lsst.pipe.base.QuantumGraphExecutionReport.write_summary_yaml
+    """
+    qgraph = QuantumGraph.loadUri(qgraph_uri)
+    butler = Butler(butler_config)
+    report = QuantumGraphExecutionReport.make_reports(butler, qgraph)
+    report.write_summary_yaml(butler, output_yaml, logs=logs)

tests/test_cliCmdReport.py

@@ -0,0 +1,89 @@
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Unit tests for ctrl_mpexec CLI update-graph-run subcommand."""
+
+import os
+import unittest
+
+import yaml
+from lsst.ctrl.mpexec.cli.pipetask import cli as pipetask_cli
+from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg
+from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir
+from lsst.pipe.base.tests.simpleQGraph import makeSimpleQGraph
+from lsst.pipe.base.tests.util import check_output_run
+from yaml.loader import SafeLoader
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+class ReportTest(unittest.TestCase):
+    """Test executing "pipetask report" command."""
+
+    def setUp(self) -> None:
+        self.runner = LogCliRunner()
+        self.root = makeTestTempDir(TESTDIR)
+
+    def tearDown(self) -> None:
+        removeTestTempDir(self.root)
+
+    def test_report(self):
+        """Test for making a report on the produced and missing expected
+        datasets in a quantum graph. in a graph.
+        """
+        metadata = {"output_run": "run"}
+        butler, qgraph = makeSimpleQGraph(
+            run="run",
+            root=self.root,
+            metadata=metadata,
+        )
+        # Check that we can get the proper run collection from the qgraph
+        self.assertEqual(check_output_run(qgraph, "run"), [])
+
+        graph_uri = os.path.join(self.root, "graph.qgraph")
+        qgraph.saveUri(graph_uri)
+
+        test_filename = os.path.join(self.root, "report_test.yaml")
+
+        result = self.runner.invoke(
+            pipetask_cli,
+            ["report", graph_uri, "-b", self.root, test_filename, "--no-logs"],
+            input="no",
+        )
+
+        # Check that we can read from the command line
+        self.assertEqual(result.exit_code, 0, clickResultMsg(result))
+
+        # Check that we can open and read the file produced by make_reports
+        with open(test_filename) as f:
+            report_output_dict = yaml.load(f, Loader=SafeLoader)
+        self.assertIsNotNone(report_output_dict["task0"])
+        self.assertIsNotNone(report_output_dict["task0"]["failed_quanta"])
+
+
+if __name__ == "__main__":
+    unittest.main()