DM-10779: Implement running time metric(s)

README.md

@@ -6,9 +6,15 @@ This package manages end-to-end testing and metric generation for the LSST DM Al
 
 ## Configuration
 
-`ap_verify` is configured from `config/dataset_config.yaml`. The file currently must have a single dictionary named `datasets`, which maps from user-visible dataset names to the eups package that implements them (see `Setting Up a Package`, below). Other configuration options may be added in the future.
+`ap_verify` is configured from `config/dataset_config.yaml`. The file currently must have:
 
-### Setting Up a Package
+* a dictionary named `datasets`, which maps from user-visible dataset names to the eups package that implements them (see `Setting Up a Dataset`, below)
+* a dictionary named `measurements`, which contains dictionaries needed for different metrics:
+    * `timing`: maps from names of metrics in the `verify_metrics` package to the Tasks or subTasks they time. The names of subTasks must be those assigned by the parent Task, and may be prefixed by the parent Task name(s) followed by a colon, as in "imageDifference:detection".
+
+Other configuration options may be added in the future.
+
+### Setting Up a Dataset
 
 `ap_verify` requires that all data be in a [dataset package](https://github.com/lsst-dm/ap_verify_dataset_template). It will create a workspace modeled after the package's `data` directory, then process any data found in the `raw` and `ref_cats` in the new workspace. Anything placed in `data` will be copied to a `ap_verify` run's workspace as-is, and must at least include a `_mapper` file naming the CameraMapper for the data.
 

bin.src/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.shebang()

bin.src/ap_verify.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+#
+# LSST Data Management System
+# Copyright 2017 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+from lsst.ap.verify import run_ap_verify
+
+if __name__ == "__main__":
+    run_ap_verify()

config/dataset_config.yaml

@@ -1,5 +1,18 @@
 ---
 datasets:
     HiTS2015: ap_verify_hits2015
+measurements:
+    timing:
+        pipe_tasks.ProcessCcdTime: processCcd
+        ip_isr.IsrTime: processCcd:isr
+        pipe_tasks.CharacterizeImageTime: processCcd:charImage
+        pipe_tasks.CalibrateTime: processCcd:calibrate
+        pipe_tasks.ImageDifferenceTime: imageDifference
+        meas_astrom.AstrometryTime: imageDifference:astrometer
+        pipe_tasks.RegisterImageTime: imageDifference:register
+        ip_diffim.ImagePsfMatchTime: imageDifference:subtract
+        meas_algorithms.SourceDetectionTime: imageDifference:detection
+        ip_diffim.DipoleFitTime: imageDifference:measurement
+        ap_association.AssociationTime: association
 ...
 

doc/conf.py

@@ -2,17 +2,17 @@
 
 from documenteer.sphinxconfig.stackconf import build_package_configs
 
-import lsst.afw
+import lsst.ap.verify
 
 _g = globals()
 _g.update(build_package_configs(
-    project_name="afw",
+    project_name="ap_verify",
     copyright="2017 Association of Univerities for "
               "Research in Astronomy, Inc.",
-    version=lsst.afw.version.__version__,
+    version=lsst.ap_verify.version.__version__,
     doxygen_xml_dirname=None))
 
 intersphinx_mapping['astropy'] = ('http://docs.astropy.org/en/stable', None)
 
 # DEBUG only
-automodsumm_writereprocessed = False
+automodsumm_writereprocessed = False

python/lsst/ap/verify/__init__.py

@@ -1 +1,5 @@
 from __future__ import absolute_import
+from .ap_verify import *
+
+import pkgutil, lsstimport
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/ap/verify/ap_verify.py

@@ -28,16 +28,17 @@
 
 from __future__ import absolute_import, division, print_function
 
-__all__ = []
+__all__ = ["run_ap_verify"]
 
 import argparse
 import os
 import re
 
 import lsst.log
-from lsst.ap.verify.dataset import Dataset
-from lsst.ap.verify.metrics import MetricsParser, check_squash_ready, AutoJob
-from lsst.ap.verify.appipe import ApPipeParser, ApPipe
+from .dataset import Dataset
+from .metrics import MetricsParser, check_squash_ready, AutoJob
+from .pipeline_driver import ApPipeParser, run_ap_pipe
+from .measurements import measure_from_metadata
 
 
 class _VerifyApParser(argparse.ArgumentParser):
@@ -125,19 +126,23 @@ def _get_output_dir(input_dir, output_arg, rerun_arg):
         return os.path.join(input_dir, "rerun", rerun_arg)
 
 
-def _measure_final_properties(metrics_job):
+def _measure_final_properties(metadata, metrics_job):
     """Measure any metrics that apply to the final result of the AP pipeline,
     rather than to a particular processing stage.
 
     Parameters
     ----------
+    metadata: `lsst.daf.base.PropertySet`
+        The metadata produced by the AP pipeline.
     metrics_job: `verify.Job`
         The Job object to which to add any metric measurements made.
     """
-    pass
+    measurements = measure_from_metadata(metadata)
+    for measurement in measurements:
+        metrics_job.measurements.insert(measurement)
 
 
-if __name__ == '__main__':
+def run_ap_verify():
     lsst.log.configure()
     log = lsst.log.Log.getLogger('ap.verify.ap_verify.main')
     # TODO: what is LSST's policy on exceptions escaping into main()?
@@ -153,6 +158,5 @@ def _measure_final_properties(metrics_job):
 
     with AutoJob(args) as job:
         log.info('Running pipeline...')
-        pipeline = ApPipe(test_data, output, args)
-        pipeline.run(job)
-        _measure_final_properties(job)
+        metadata = run_ap_pipe(test_data, output, args, job)
+        _measure_final_properties(metadata, job)

python/lsst/ap/verify/config.py

@@ -0,0 +1,66 @@
+#
+# LSST Data Management System
+# Copyright 2017 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+from __future__ import absolute_import, division, print_function
+
+from future.utils import raise_from
+
+from lsst.daf.persistence import Policy
+
+
+class Config(object):
+    """Confuration manager for ap_verify.
+
+    This is a singleton Policy that may be accessed from other modules in
+    ap_verify as needed using `Config.instance`. Please do not construct
+    objects of this class directly.
+    """
+
+    def __init__(self):
+        path = Policy.defaultPolicyFile('ap_verify', 'dataset_config.yaml', 'config')
+        self._all_info = Policy(path)
+        self._validate()
+
+    def _validate(self):
+        """Tests that the loaded configuration is correct, and raises
+        RuntimeError otherwise.
+        """
+        try:
+            dataset_map = self._all_info['datasets']
+            if not isinstance(dataset_map, Policy):
+                raise TypeError('`datasets` is not a dictionary')
+        except (KeyError, TypeError) as e:
+            raise_from(RuntimeError('Invalid config file.'), e)
+
+        try:
+            measurement_map = self._all_info['measurements']
+            if not isinstance(measurement_map, Policy):
+                raise TypeError('`measurements` is not a dictionary')
+            timing_map = measurement_map['timing']
+            if not isinstance(timing_map, Policy):
+                raise TypeError('`measurements.timing` is not a dictionary')
+        except (KeyError, TypeError) as e:
+            raise_from(RuntimeError('Invalid config file.'), e)
+
+
+# Hack, but I don't know how else to make Config.instance act like a dictionary of config options
+Config.instance = Config()._all_info