tickets/dm-7308 Port obs_sdss to Python 3

.gitignore

@@ -8,6 +8,7 @@ config.log
 *.cfgc
 *.pyc
 *_wrap.cc
+.cache
 *Lib.py
 doc/html
 doc/*.tag

bin.src/genCoaddRegistry.py

@@ -1,27 +1,28 @@
 #!/usr/bin/env python
 
-# 
+#
 # LSST Data Management System
 # Copyright 2008, 2009, 2010 LSST Corporation.
-# 
+#
 # This product includes software developed by the
 # LSST Project (http://www.lsst.org/).
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
-# You should have received a copy of the LSST License Statement and 
-# the GNU General Public License along with this program.  If not, 
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
+from __future__ import print_function
 import glob
 from optparse import OptionParser
 import os
@@ -36,13 +37,14 @@
 import lsst.afw.image as afwImage
 import lsst.skypix as skypix
 
+
 def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
     if os.path.exists(outputRegistry):
-        print >>sys.stderr, "Output registry exists; will not overwrite."
+        print("Output registry exists; will not overwrite.", file=sys.stderr)
         sys.exit(1)
     if inputRegistry is not None:
         if not os.path.exists(inputRegistry):
-            print >>sys.stderr, "Input registry does not exist."
+            print("Input registry does not exist.", file=sys.stderr)
             sys.exit(1)
         shutil.copy(inputRegistry, outputRegistry)
 
@@ -71,22 +73,23 @@ def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
             for filterDir in glob.iglob(os.path.join(dir, "*")):
                 processBand(filterDir, conn, done, qsp)
     finally:
-        print >>sys.stderr, "Cleaning up..."
+        print("Cleaning up...", file=sys.stderr)
         conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
         conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
         conn.commit()
         conn.close()
 
+
 def processBand(filterDir, conn, done, qsp):
     nProcessed = 0
     nSkipped = 0
     nUnrecognized = 0
-    print >>sys.stderr, filterDir, "... started"
+    print(filterDir, "... started", file=sys.stderr)
     for fits in glob.iglob(
             os.path.join(filterDir, "fpC*_ts_coaddNorm_NN.fit.gz")):
         m = re.search(r'/([ugriz])/fpC-(\d{6})-\1(\d)-(\d{4})_ts_coaddNorm_NN.fit.gz', fits)
         if not m:
-            print >>sys.stderr, "Warning: Unrecognized file:", fits
+            print("Warning: Unrecognized file:", fits, file=sys.stderr)
             nUnrecognized += 1
             continue
 
@@ -95,43 +98,43 @@ def processBand(filterDir, conn, done, qsp):
         run = int(run)
         field = int(field)
         key = "%d_B%s_C%d_F%d" % (run, filter, camcol, field)
-        if done.has_key(key):
+        if key in done:
             nSkipped += 1
             continue
 
         md = afwImage.readMetadata(fits)
         conn.execute("""INSERT INTO raw VALUES
             (NULL, ?, ?, ?, ?)""", (run, filter, camcol, field))
-   
+
         for row in conn.execute("SELECT last_insert_rowid()"):
             id = row[0]
             break
 
         wcs = afwImage.makeWcs(md)
         poly = skypix.imageToPolygon(wcs,
-                md.get("NAXIS1"), md.get("NAXIS2"),
-                padRad=0.000075) # about 15 arcsec
+                                     md.get("NAXIS1"), md.get("NAXIS2"),
+                                     padRad=0.000075)  # about 15 arcsec
         pix = qsp.intersect(poly)
         for skyTileId in pix:
             conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
-                    (id, skyTileId))
+                         (id, skyTileId))
 
         nProcessed += 1
         if nProcessed % 100 == 0:
             conn.commit()
 
     conn.commit()
-    print >>sys.stderr, filterDir, \
-            "... %d processed, %d skipped, %d unrecognized" % \
-            (nProcessed, nSkipped, nUnrecognized)
+    print(filterDir, \
+        "... %d processed, %d skipped, %d unrecognized" % \
+        (nProcessed, nSkipped, nUnrecognized), file=sys.stderr)
 
 if __name__ == "__main__":
     parser = OptionParser(usage="""%prog [options] DIR ...
 
 DIR should contain a directory per filter containing coadd pieces.""")
     parser.add_option("-i", dest="inputRegistry", help="input registry")
     parser.add_option("-o", dest="outputRegistry", default="registry.sqlite3",
-            help="output registry (default=registry.sqlite3)")
+                      help="output registry (default=registry.sqlite3)")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         parser.error("Missing directory argument(s)")

bin.src/genInputRegistry.py

@@ -1,27 +1,28 @@
 #!/usr/bin/env python
 
-# 
+#
 # LSST Data Management System
 # Copyright 2008, 2009, 2010 LSST Corporation.
-# 
+#
 # This product includes software developed by the
 # LSST Project (http://www.lsst.org/).
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
-# You should have received a copy of the LSST License Statement and 
-# the GNU General Public License along with this program.  If not, 
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 
+from __future__ import print_function
 import glob
 from optparse import OptionParser
 import os
@@ -37,13 +38,14 @@
 import lsst.afw.image as afwImage
 import lsst.skypix as skypix
 
+
 def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
     if os.path.exists(outputRegistry):
-        print >>sys.stderr, "Output registry exists; will not overwrite."
+        print("Output registry exists; will not overwrite.", file=sys.stderr)
         sys.exit(1)
     if inputRegistry is not None:
         if not os.path.exists(inputRegistry):
-            print >>sys.stderr, "Input registry does not exist."
+            print("Input registry does not exist.", file=sys.stderr)
             sys.exit(1)
         shutil.copy(inputRegistry, outputRegistry)
 
@@ -76,24 +78,25 @@ def process(dirList, inputRegistry, outputRegistry="registry.sqlite3"):
             else:
                 processRun(dir, conn, done, qsp)
     finally:
-        print >>sys.stderr, "Cleaning up..."
+        print("Cleaning up...", file=sys.stderr)
         conn.execute("""CREATE UNIQUE INDEX uq_raw ON raw
                 (run, filter, camcol, field)""")
         conn.execute("CREATE INDEX ix_skyTile_id ON raw_skyTile (id)")
         conn.execute("CREATE INDEX ix_skyTile_tile ON raw_skyTile (skyTile)")
         conn.commit()
         conn.close()
 
+
 def processRun(runDir, conn, done, qsp):
     nProcessed = 0
     nSkipped = 0
     nUnrecognized = 0
-    print >>sys.stderr, runDir, "... started"
+    print(runDir, "... started", file=sys.stderr)
     for fits in glob.iglob(
             os.path.join(runDir, "*", "corr", "[1-6]", "fpC*.fit.gz")):
         m = re.search(r'(\d+)/corr/([1-6])/fpC-(\d{6})-([ugriz])\2-(\d{4}).fit.gz', fits)
         if not m:
-            print >>sys.stderr, "Warning: Unrecognized file:", fits
+            print("Warning: Unrecognized file:", fits, file=sys.stderr)
             nUnrecognized += 1
             continue
 
@@ -103,7 +106,7 @@ def processRun(runDir, conn, done, qsp):
         run = int(run)
         field = int(field)
         key = "%d_R%d_B%s_C%d_F%d" % (run, rerun, filter, camcol, field)
-        if done.has_key(key) or rerun < 40:
+        if key in done or rerun < 40:
             nSkipped += 1
             continue
 
@@ -118,36 +121,36 @@ def processRun(runDir, conn, done, qsp):
         seconds = float(second)
         second = int(seconds)
         taiObs = dafBase.DateTime(int(year), int(month), int(day), int(hour),
-                int(minute), second, dafBase.DateTime.TAI)
+                                  int(minute), second, dafBase.DateTime.TAI)
         taiObs = dafBase.DateTime(taiObs.nsecs() +
-                long((seconds - second) * 1000000000), dafBase.DateTime.TAI)
+                                  int((seconds - second) * 1000000000), dafBase.DateTime.TAI)
         taiObs = taiObs.toString(dafBase.DateTime.UTC)[:-1]
         strip = "%d%s" % (md.get('STRIPE'), md.get('STRIP'))
         conn.execute("""INSERT INTO raw VALUES
             (NULL, ?, ?, ?, ?, ?, ?, ?)""",
-            (run, rerun, filter, camcol, field, taiObs, strip))
-   
+                     (run, rerun, filter, camcol, field, taiObs, strip))
+
         for row in conn.execute("SELECT last_insert_rowid()"):
             id = row[0]
             break
 
         wcs = afwImage.makeWcs(md)
         poly = skypix.imageToPolygon(wcs,
-                md.get("NAXIS1"), md.get("NAXIS2"),
-                padRad=0.000075) # about 15 arcsec
+                                     md.get("NAXIS1"), md.get("NAXIS2"),
+                                     padRad=0.000075)  # about 15 arcsec
         pix = qsp.intersect(poly)
         for skyTileId in pix:
             conn.execute("INSERT INTO raw_skyTile VALUES(?, ?)",
-                    (id, skyTileId))
+                         (id, skyTileId))
 
         nProcessed += 1
         if nProcessed % 100 == 0:
             conn.commit()
 
     conn.commit()
-    print >>sys.stderr, runDir, \
-            "... %d processed, %d skipped, %d unrecognized" % \
-            (nProcessed, nSkipped, nUnrecognized)
+    print(runDir, \
+        "... %d processed, %d skipped, %d unrecognized" % \
+        (nProcessed, nSkipped, nUnrecognized), file=sys.stderr)
 
 if __name__ == "__main__":
     parser = OptionParser(usage="""%prog [options] DIR ...
@@ -156,7 +159,7 @@ def processRun(runDir, conn, done, qsp):
 or a visit subdirectory.""")
     parser.add_option("-i", dest="inputRegistry", help="input registry")
     parser.add_option("-o", dest="outputRegistry", default="registry.sqlite3",
-            help="output registry (default=registry.sqlite3)")
+                      help="output registry (default=registry.sqlite3)")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         parser.error("Missing directory argument(s)")

config/assembleCoadd.py

@@ -5,17 +5,16 @@
 config.select.retarget(SelectSdssImagesTask)
 config.scaleZeroPoint.retarget(ScaleSdssZeroPointTask)
 
-config.matchBackgrounds.usePolynomial=True
-config.matchBackgrounds.binSize=128
-config.matchBackgrounds.order=4
-config.subregionSize=(2500, 2500)
-config.sigmaClip=5
-config.maxMatchResidualRatio=1.7
-config.maxMatchResidualRMS=1.0
-config.autoReference=False
-
-#Configs for deep coadd
-config.coaddName='deep'
-config.select.maxFwhm=2.0
-config.select.quality=2
+config.matchBackgrounds.usePolynomial = True
+config.matchBackgrounds.binSize = 128
+config.matchBackgrounds.order = 4
+config.subregionSize = (2500, 2500)
+config.sigmaClip = 5
+config.maxMatchResidualRatio = 1.7
+config.maxMatchResidualRMS = 1.0
+config.autoReference = False
 
+# Configs for deep coadd
+config.coaddName = 'deep'
+config.select.maxFwhm = 2.0
+config.select.quality = 2

config/makeCoaddTempExp.py

@@ -3,9 +3,9 @@
 
 config.select.retarget(SelectSdssImagesTask)
 
-config.doOverwrite=True
+config.doOverwrite = True
 
-#Configs for deep coadd
-config.coaddName='deep'
-config.select.maxFwhm=2.0
-config.select.quality=2
+# Configs for deep coadd
+config.coaddName = 'deep'
+config.select.maxFwhm = 2.0
+config.select.quality = 2

config/mergeCoaddDetections.py

@@ -1 +1 @@
-config.priorityList = ["r", "i", "g", "z", "u",]
+config.priorityList = ["r", "i", "g", "z", "u", ]

config/mergeCoaddMeasurements.py

@@ -1 +1 @@
-config.priorityList = ["r", "i", "g", "z", "u",]
+config.priorityList = ["r", "i", "g", "z", "u", ]

config/sourceAssoc.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 config.inputLevel = "filter"
 config.inputSourceDataset = "src"
 config.inputCalexpMetadataDataset = "calexp_md"
@@ -7,4 +8,4 @@
     config.measSlots.modelFlux = "multishapelet.combo.flux"
 except ImportError:
     # TODO: find a better way to log this
-    print "WARNING: Could not import lsst.meas.extensions.multiShapelet; model fluxes not enabled!"
+    print("WARNING: Could not import lsst.meas.extensions.multiShapelet; model fluxes not enabled!")