DM-4848: Measure photometric repeatability and correctness of reported errors

README.md

@@ -1,6 +1,6 @@
 A set of utilities to run the processCcd task on some 
 CFHT data and DECam data
-and validate the astrometry of the results.
+and validate the astrometric and photometric repeatability of the results.
 
 Pre-requisites: install and declare the following
 1. `pipe_tasks` from the LSST DM stack (note that pipe_tasks is included with lsst_apps, which is the usual thing to install)
@@ -12,7 +12,7 @@ Pre-requisites: install and declare the following
 The `obs_decam`, `obs_cfht`, `validation_data_cfht`, `validation_data_decam`, `validate_drp` products are also buildable by the standard LSST DM stack tools: `lsstsw` or `eups distrib`.  But they (intentionally) aren't in the dependency tree of `lsst_apps`.  If you have a stack already installed with `lsst_apps`, you can install these in the same manner.  E.g.,
 
 ```
-eups distrib obs_decam obs_cfht validation_data_decam validation_data_cfht validate_drp
+eups distrib install obs_decam obs_cfht validation_data_decam validation_data_cfht validate_drp
 ```
 
 XOR
@@ -100,8 +100,8 @@ Once these basic steps are completed, then you can run any of the following:
 
 Note that the example validation test selects several of the CCDs and will fail if you just pass it a repository with 1 visit or just 1 CCD.
 
-Files :
--------
+Files of Interest:
+------------------
 * `examples/runCfhtTest.sh`  : CFHT Run initialization, ingest, measurement, and astrometry validation.
 * `examples/runDecamTest.sh` : DECam Run initialization, ingest, measurement, and astrometry validation.
 * `examples/validateCfht.py`    : CFHT run some analysis on the output data produced by processCcd.py
@@ -110,4 +110,11 @@ Files :
 * `examples/runDecam.list`   : DECam list of vistits / ccd to be processed by processCcd
 * `config/newAstrometryConfig.py`  : configuration for running processCcd with the new AstrometryTask
 * `config/anetAstrometryConfig.py` : configuration for running processCcd ANetAstrometryTask
+* `bin.src/validateCfht.py` : Wrapper script to analyze CFHT data in validation_data_cfht
+* `bin.src/validateDecam.py` : Wrapper script to analyze DECam data in validation_data_decam
+* `bin.src/validateDecamCosmos.py` : Wrapper script to analyze larger set of DECam data taken in the COSMOS field.
+* `bin.src/validateCfht.py` : Wrapper script to analyze CFHT data in validation_data_cfht
+* `python/lsst/validate/drp/calcSrd.py` : calculate metrics defined by the LSST SRC.
+* `python/lsst/validate/drp/plotAstrometryPhotometry.py` : plotting routines
+* `python/lsst/validate/drp/checkAstrometryPhotometry.py` : coordination and calculation routines.
 * `README.md` : THIS FILE.  Guide and examples.

bin.src/validateCfht.py

@@ -25,15 +25,12 @@
 import os.path
 import sys
 
-from lsst.validate.drp import checkAstrometry
+from lsst.validate.drp import checkAstrometryPhotometry
 
 
 def defaultData(repo):
     # List of visits to be considered
-    visits = [850587]
-
-    # Reference visit (the other viisits will be compared to this one
-    ref = 849375
+    visits = [849375, 850587]
 
     # List of CCD to be considered (source calalogs will be concateneted)
     ccd = [12, 13, 14, 21, 22, 23]
@@ -42,13 +39,12 @@ def defaultData(repo):
     # Reference values for the median astrometric scatter and the number of matches
     good_mag_limit = 21.0
     medianRef = 25
-    matchRef = 5600
+    matchRef = 5000
 
-    visitDataIds = [[{'visit': v, 'filter': filter, 'ccd': c} for v in visits]
+    visitDataIds = [{'visit': v, 'filter': filter, 'ccd': c} for v in visits
                     for c in ccd]
-    refDataIds = [{'visit': ref, 'filter': filter, 'ccd': c} for c in ccd]
 
-    return visitDataIds, refDataIds, good_mag_limit, medianRef, matchRef
+    return visitDataIds, good_mag_limit, medianRef, matchRef
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
@@ -62,5 +58,5 @@ def defaultData(repo):
         print("Could not find repo %r" % (repo,))
         sys.exit(1)
 
-    visitDataIds, refDataIds, good_mag_limit, medianRef, matchRef = defaultData(repo)
-    checkAstrometry.run(repo, visitDataIds, refDataIds, good_mag_limit, medianRef, matchRef)
+    args = defaultData(repo)
+    checkAstrometryPhotometry.run(repo, *args)

bin.src/validateDecam.py

@@ -25,15 +25,12 @@
 import os.path
 import sys
 
-from lsst.validate.drp import checkAstrometry
+from lsst.validate.drp import checkAstrometryPhotometry
 
 
 def defaultData(repo):
     # List of visits to be considered
-    visits = [176846]
-
-    # Reference visit (the other visits will be compared to this one)
-    ref = 176837
+    visits = [176837, 176846]
 
     # List of CCD to be considered (source catalogs will be concateneted)
     ccd = [10, 11, 12, 13, 14, 15, 16, 17, 18]
@@ -45,11 +42,10 @@ def defaultData(repo):
     medianRef = 25  # [arcsec]
     matchRef = 10000  # [number of stars]
 
-    visitDataIds = [[{'visit': v, 'filter': filter, 'ccdnum': c} for v in visits]
+    visitDataIds = [{'visit': v, 'filter': filter, 'ccdnum': c} for v in visits
                     for c in ccd]
-    refDataIds = [{'visit': ref, 'filter': filter, 'ccdnum': c} for c in ccd]
 
-    return visitDataIds, refDataIds, good_mag_limit, medianRef, matchRef
+    return visitDataIds, good_mag_limit, medianRef, matchRef
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
@@ -64,4 +60,4 @@ def defaultData(repo):
         sys.exit(1)
 
     args = defaultData(repo)
-    checkAstrometry.run(repo, *args)
+    checkAstrometryPhotometry.run(repo, *args)

bin.src/validateDecamCosmos.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+# LSST Data Management System
+# Copyright 2008-2016 AURA/LSST.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <https://www.lsstcorp.org/LegalNotices/>.
+
+from __future__ import print_function
+
+import os.path
+import sys
+
+from lsst.validate.drp import checkAstrometryPhotometry
+
+
+def defaultData(repo):
+    # List of visits to be considered
+    visits = [176837, 176839, 176840, 176841, 176842, 176843, 176844, 176845, 176846]
+
+    # List of CCD to be considered (source catalogs will be concateneted)
+    ccd = [10, 11, 12, 13, 14, 15, 16, 17, 18]
+    filter = 'z'
+
+    # Reference values that the DECam analysis should pass
+    #  for the median astrometric scatter and the number of matches
+    good_mag_limit = 21  # [mag]
+    medianRef = 25  # [arcsec]
+    matchRef = 10000  # [number of stars]
+
+    visitDataIds = [{'visit': v, 'filter': filter, 'ccdnum': c} for v in visits
+                    for c in ccd]
+
+    return visitDataIds, good_mag_limit, medianRef, matchRef
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("""Usage: valid_cosmos repo
+where repo is the path to a repository containing the output of processCcd
+""")
+        sys.exit(1)
+
+    repo = sys.argv[1]
+    if not os.path.isdir(repo):
+        print("Could not find repo %r" % (repo,))
+        sys.exit(1)
+
+    args = defaultData(repo)
+    checkAstrometryPhotometry.run(repo, *args)