DM-14806: bring demo to standards

.gitignore

@@ -1,5 +1,17 @@
+.cache
+pytest_session.txt
 output
 output_small
 detected-sources.txt
 detected-sources_small.txt
 astrometry_sdss.png
+bin
+
+.coverage
+.sconf_temp/
+.sconsign.dblite
+config.log
+python/lsst/lsst/dm/stack/demo/version.py
+tests/.tests/
+tests/__pycache__/
+ups/__pycache__/

.travis.yaml

@@ -0,0 +1,8 @@
+sudo: false
+language: python
+matrix:
+  include:
+    - python: '3.6'
+      install:
+      - pip install flake8
+      script: flake8

README

@@ -14,7 +14,7 @@ Usage
 To run the example, do::
 
 	$ source $LSST_HOME/loadLSST.sh
-	$ setup obs_sdss
+	$ setup -r .
 	$ ./bin/demo.sh  (or: "./bin/demo.sh --small"  if local memory is constrained)
 
 where $LSST_HOME is the directory where your LSST DM stack resides.
@@ -35,7 +35,7 @@ output; choose a result set appropriate for your system. The ./bin/compare
 script can then be used to check if your results are within the expected
 numerical tolerances::
 
-    $ python bin/compare detected-sources.txt
+    $ python bin/compare.py detected-sources.txt
     Ok.
 
 If you use the "--small" run option then the corresponding output file is
@@ -44,7 +44,7 @@ called "output_small".
 
 Check the astrometric relative RMS with::
 
-    $ python bin/check_astrometry output
+    $ python bin/check_astrometry.py output
 
 Included Data
 -------------

SConstruct

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConstruct("lsst_dm_stack_demo", disableCc=True)

bin.src/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.shebang()

bin/check_astrometry → bin.src/check_astrometry.py

@@ -26,7 +26,6 @@
 import os.path
 import sys
 
-import matplotlib.pylab as plt
 import numpy as np
 
 import lsst.daf.persistence as dafPersist
@@ -138,22 +137,26 @@ def loadAndMatchData(repo, visits, fields, ref, ref_field, camcol, filter):
             did = {'run': ref, 'filter': filter, 'field': ref_field, 'camcol': camcolRef}
             md = butler.get("calexp_md", did, immediate=True)
             calib = afwImage.Calib(md)
+            calib.setThrowOnNegativeFlux(False)
             # compute magnitude
             refMag = calib.getMagnitude(mRef.get('base_PsfFlux_flux'))
 
             mag.append(refMag)
             dist.append(ang)
 
     return pipeBase.Struct(
-        mag = mag,
-        dist = dist,
-        match = matchNum
+        mag=mag,
+        dist=dist,
+        match=matchNum
     )
 
 
 def plotAstrometry(mag, dist, match, good_mag_limit=19.5):
     """Plot angular distance between matched sources from different exposures."""
 
+    # Defer importing of matplotlib until we need it.
+    import matplotlib.pylab as plt
+
     plt.rcParams['axes.linewidth'] = 2
     plt.rcParams['mathtext.default'] = 'regular'
 
@@ -208,7 +211,9 @@ def checkAstrometry(mag, dist, match,
     @param medianRef  Median reference astrometric scatter in arcseconds.
     @param matchRef   Should match at least matchRef stars.
 
-    Return the astrometric scatter (RMS, arcsec) for all good stars.
+    Return a boolean indicating whether the astrometric scatter was less than
+    the supplied reference, and the astrometric scatter (RMS, arcsec) for all
+    good stars.
 
     Notes:
        The scatter and match defaults are appropriate to SDSS are stored here.
@@ -225,23 +230,36 @@ def checkAstrometry(mag, dist, match,
     if astromScatter > medianRef:
         print("Median astrometric scatter %.1f mas is larger than reference : %.1f mas " %
               (astromScatter, medianRef))
+        passed = False
+    else:
+        passed = True
     if match < matchRef:
-        print("Number of matched sources %d is too small (shoud be > %d)" % (match, matchRef))
+        print("Number of matched sources %d is too small (should be > %d)" % (match, matchRef))
+        passed = False
 
-    return astromScatter
+    return passed, astromScatter
 
 
-def main(repo, runs, fields, ref, ref_field, camcol, filter):
+def main(repo, runs, fields, ref, ref_field, camcol, filter, plot=False):
     """Main executable.
 
+    Returns True if the test passed, False otherwise.
     """
 
     struct = loadAndMatchData(repo, runs, fields, ref, ref_field, camcol, filter)
     mag = struct.mag
     dist = struct.dist
     match = struct.match
-    checkAstrometry(mag, dist, match)
-    plotAstrometry(mag, dist, match)
+
+    # Limit depends on filter
+    medianRef = 100
+    if filter == 'i':
+        medianRef = 105
+
+    passed, astromScatter = checkAstrometry(mag, dist, match, medianRef=medianRef)
+    if plot:
+        plotAstrometry(mag, dist, match)
+    return passed
 
 
 def defaultData(repo):
@@ -255,7 +273,7 @@ def defaultData(repo):
 
     # List of camcol to be considered (source calalogs will be concateneted)
     camcol = [4]
-    filter = 'r'
+    filter = 'i'
 
     return runs, fields, ref, ref_field, camcol, filter
 
@@ -273,4 +291,8 @@ def defaultData(repo):
         sys.exit(1)
 
     runs, fields, ref, ref_field, camcol, filter = defaultData(repo)
-    main(repo, runs, fields, ref, ref_field, camcol, filter)
+    passed = main(repo, runs, fields, ref, ref_field, camcol, filter)
+    if passed:
+        print("Ok.")
+    else:
+        sys.exit(1)

bin/demo.sh

@@ -64,7 +64,7 @@ processCcd.py input --id run=4192 filter=$FILTER_SET_4192 camcol=4 field=300 --i
 #   on modern OS X versions.
 # The `#!/usr/bin/env python` in the first line of export-results
 #   no longer loads the correct environment.
-python ./bin/export-results output$SIZE_EXT > detected-sources$SIZE_EXT.txt
+python ./bin/export-results.py output$SIZE_EXT > detected-sources$SIZE_EXT.txt
 
 echo
 echo "Processing completed successfully. The results are in detected-sources$SIZE_EXT.txt."

setup.cfg

@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 110
+ignore = E133, E226, E228, N802, N803, N806
+exclude = andConfig.py, processCcd.py
+
+[tool:pytest]
+addopts = --flake8
+flake8-ignore = E133 E226 E228 N802 N803 N806
+    processCcd.py ALL
+    andConfig.py ALL

tests/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.tests(pyList=[])

tests/test_demo.py

@@ -0,0 +1,53 @@
+#
+# LSST Data Management System
+# Copyright 2012-2017 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import unittest
+import os
+
+import lsst.utils.tests
+from lsst.utils import getPackageDir
+
+package_root = getPackageDir('lsst_dm_stack_demo')
+
+executable_dir = os.path.join(package_root, 'bin')
+
+
+class DemoTestCase(lsst.utils.tests.ExecutablesTestCase):
+    """Test the demo scrpts for executablility."""
+    def testDemo(self):
+        """Test demo"""
+        self.assertExecutable("demo.sh",
+                              root_dir=executable_dir,
+                              args=["--small"],
+                              msg="Running demo failed")
+        self.assertExecutable("compare.py",
+                              root_dir=executable_dir,
+                              args=['detected-sources_small.txt'],
+                              msg="Compare failed")
+        self.assertExecutable("check_astrometry.py",
+                              root_dir=executable_dir,
+                              args=['output_small'],
+                              msg="Check astrometry failed")
+
+
+if __name__ == "__main__":
+    unittest.main()

ups/lsst_dm_stack_demo.cfg

@@ -0,0 +1,13 @@
+# -*- python -*-
+
+import lsst.sconsUtils
+
+dependencies = dict(
+)
+
+config = lsst.sconsUtils.Configuration(
+    __file__,
+    libs=[],
+    hasDoxygenInclude=False,
+    hasSwigFiles=False,
+)

ups/lsst_dm_stack_demo.table

@@ -0,0 +1,6 @@
+setupRequired(python)
+setupRequired(utils)
+setupRequired(sconsUtils)
+setupRequired(obs_sdss)
+
+envPrepend(PATH, ${PRODUCT_DIR}/bin)