DM-16828: Add Job viewer to lsst.verify

bin.src/inspect_job.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+#
+# This file is part of ap_verify.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Print the contents of a persisted Job object.
+
+This script takes as arguments one or more Job .json files, and prints the
+top-level metadata and a summary of any measurements. It does not print
+metrics that don't have measurements (there are far too many) or
+specifications (which are not helpful for testing measurement code).
+"""
+
+import sys
+
+from lsst.verify.bin.inspectjob import main
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Syntax: %s <job file> [[job file]...]" % sys.argv[0])
+        sys.exit(1)
+    main(sys.argv[1:])

doc/lsst.verify/index.rst

@@ -14,6 +14,14 @@ Measurements made through `lsst.verify` can be uploaded to LSST's SQUASH_ monito
 
    `SQR-019: LSST Verification Framework API Demonstration <https://sqr-019.lsst.io>`_.
 
+Using lsst.verify
+=================
+
+.. toctree::
+   :maxdepth: 1
+
+   inspect_job
+
 Python API reference
 ====================
 

doc/lsst.verify/inspect_job.rst

@@ -0,0 +1,67 @@
+.. _lsst.verify.inspect_job:
+
+#########################################
+Reviewing Job objects with inspect_job.py
+#########################################
+
+`lsst.verify.Job` provides a rich interface for examining verification jobs, including pass/fail notifications for specifications (see `SQR-019`_).
+However, this interface is often too heavyweight for debugging purposes.
+:command:`inspect_job.py` is a command-line tool that lets the developer quickly see what information is being stored in a `~lsst.verify.Job` without performing additional analysis.
+
+.. _SQR-019: https://sqr-019.lsst.io
+
+Usage
+=====
+
+:command:`inspect_job.py` takes a list of :file:`.json` files, and no other arguments.
+Operating system wildcards can be used.
+For example:
+
+.. prompt:: bash
+
+   inspect_job.py package*.verify.json global.verify.json
+
+Output
+======
+
+:command:`inspect_job.py` prints a report for each file passed to it.
+
+The report starts with the job-level metadata in the file, formatted as one key-value pair per line.
+Unlike `lsst.verify.Job.meta`, this does not include metadata associated with individual measurements.
+Presenting job and measurement metadata separately makes it easier for developers to see what metadata are added at which level.
+
+The report then lists all the measurements in the file, formatted as one pair of metric and value per line.
+If there are metadata associated with a measurement, they are listed after the value.
+
+:command:`inspect_job.py` does *not* report specifications, which are usually irrelevant to the code that creates measurements or jobs.
+It also does not report metrics that don't have a corresponding measurement, as the list could be cluttered with many metrics that don't apply to the task being instrumented.
+
+An example report looks like:
+
+.. code-block:: none
+
+   Common metadata:
+                                       ccdnum = 42
+                                        visit = 411657
+                                       object = Blind15A_26
+                                         date = 2015-02-19
+                                   instrument = DECAM
+                                       filter = g
+
+   Measurements:
+                      ip_diffim.numSciSources =  1326.0 ct
+         ip_diffim.fracDiaSourcesToSciSources =     0.0385
+   ap_association.totalUnassociatedDiaObjects =   540.0 ct
+                           ap_pipe.ApPipeTime =  63.1475 s ({'estimator': 'pipe.base.timeMethod'})
+                    pipe_tasks.ProcessCcdTime =  24.3298 s ({'estimator': 'pipe.base.timeMethod'})
+                               ip_isr.IsrTime =   0.9623 s ({'estimator': 'pipe.base.timeMethod'})
+             pipe_tasks.CharacterizeImageTime =   7.5473 s ({'estimator': 'pipe.base.timeMethod'})
+                     pipe_tasks.CalibrateTime =  11.0519 s ({'estimator': 'pipe.base.timeMethod'})
+               pipe_tasks.ImageDifferenceTime =  37.5217 s ({'estimator': 'pipe.base.timeMethod'})
+          meas_algorithms.SourceDetectionTime =   1.0205 s ({'estimator': 'pipe.base.timeMethod'})
+                      ip_diffim.DipoleFitTime =   1.9112 s ({'estimator': 'pipe.base.timeMethod'})
+               ap_association.AssociationTime =   0.6594 s ({'estimator': 'pipe.base.timeMethod'})
+              ap_association.numNewDiaObjects =    51.0 ct
+         ap_association.fracUpdatedDiaObjects =        0.0
+     ap_association.numUnassociatedDiaObjects =     0.0 ct
+

python/lsst/verify/bin/inspectjob.py

@@ -0,0 +1,126 @@
+# This file is part of ap_verify.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Print the contents of a persisted Job object.
+
+This script takes as arguments one or more Job .json files, and prints the
+top-level metadata and a summary of any measurements. It does not print
+metrics that don't have measurements (there are far too many) or
+specifications (which are not helpful for testing measurement code).
+"""
+
+__all__ = ["main", "inspect_job"]
+
+import json
+
+from lsst.verify import Job
+
+
+def _is_measurement_metadata(key, metrics):
+    """Test whether a job-level metadata key is really measurement metadata.
+
+    Parameters
+    ----------
+    key : `str`
+        The metadata key to test.
+    metrics : iterable of `lsst.verify.Name` or of `str`
+        The metrics recorded in the job.
+
+    Returns
+    -------
+    result : `bool`
+        `True` if ``key`` represents measurement metadata, `False` if it
+        represents purely job-level metadata.
+    """
+    for metric in metrics:
+        if str(metric) in key:
+            return True
+    return False
+
+
+def _simplify_key(key, prefix):
+    """Remove a prefix from a key, if it's present.
+
+    Parameters
+    ----------
+    key : `str`
+        The key to simplify.
+    prefix : `str`
+        The prefix to remove from ``key``.
+
+    Returns
+    -------
+    simplifiedKey : `str`
+        ``key`` with any initial ``prefix`` removed
+    """
+    if key.startswith(prefix):
+        return key.replace(prefix, "", 1)
+    else:
+        return key
+
+
+def inspect_job(job):
+    """Present the measurements in a Job object.
+
+    The measurements and any metadata are printed to standard output.
+
+    Parameters
+    ----------
+    job : `lsst.verify.Job`
+        The Job to examine.
+    """
+    # Leave enough space for output so that all '=' characters are aligned
+    max_metric_length = max([len(str(metric)) for metric in job.measurements])
+
+    print("Common metadata:")
+    for key, value in job.meta.items():
+        if _is_measurement_metadata(key, job.measurements.keys()):
+            continue
+        print("%*s = %s" % (max_metric_length, key, value))
+
+    print("\nMeasurements:")
+    for metric, measurement in job.measurements.items():
+        pretty_quantity = measurement.quantity.round(4)
+        if measurement.notes:
+            prefix = str(measurement.metric_name) + "."
+            # Raw representation of measurement.notes hard to read
+            simple_notes = {_simplify_key(key, prefix): value
+                            for key, value in measurement.notes.items()}
+            print("%*s = %10s (%s)"
+                  % (max_metric_length, metric, pretty_quantity, simple_notes))
+        else:
+            print("%*s = %10s" % (max_metric_length, metric, pretty_quantity))
+
+
+def main(filenames):
+    """Present all Job files.
+
+    Parameters
+    ----------
+    filenames : `list` of `str`
+        The Job files to open. Must be in JSON format.
+    """
+    for filename in filenames:
+        if len(filenames) > 1:
+            print("\n%s:" % filename)
+        with open(filename) as f:
+            job = Job.deserialize(**json.load(f))
+        inspect_job(job)

python/lsst/verify/blob.py

@@ -91,8 +91,8 @@ def deserialize(cls, identifier=None, name=None, data=None):
         blob : `Blob`
             The `Blob` instance deserialied from a blob JSON object.
 
-        Example
-        -------
+        Examples
+        --------
         This class method is designed to roundtrip JSON objects created a
         Blob instance. For example:
 

python/lsst/verify/compatibility/metricsControllerTask.py

@@ -58,6 +58,7 @@ class MetricsControllerTask(Task):
 
     Because ``MetricsControllerTask`` cannot support the full functionality of
     the Gen 3 framework, it places several restrictions on its metrics:
+
         * no ``MetricTask`` may depend on the output of another ``MetricTask``
         * the granularity of the metrics is determined by the inputs to
           ``runDataRefs``; configuration information specifying a different