DM-18051: Get defect machinery working for the AuxTel

.gitignore

@@ -11,3 +11,5 @@ python/lsst/obs/lsst/version.py
 .pytest_cache
 tests/.tests
 pytest_session.txt
+*.fits
+*.sqlite3

SConstruct

@@ -1,4 +1,11 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
-scripts.BasicSConstruct("obs_lsst", disableCc=True,
-                        defaultTargets=("shebang", "policy",) + scripts.DEFAULT_TARGETS)
+
+# Note the ordering here is critical. AuxTel is put at the end here to ensure
+# that the tests are run first and version.py is created, because creation of
+# of the defect registry required the camera to be instantiated.
+# If other cameras add defect generation they should add their build to
+# the end of this list, along with auxTel
+targetList = ("version", "shebang", "policy",) + scripts.DEFAULT_TARGETS + ("auxTel",)
+
+scripts.BasicSConstruct("obs_lsst", disableCc=True, defaultTargets=targetList)

auxTel/SConscript

@@ -0,0 +1,30 @@
+# -*- python -*-
+
+import lsst.sconsUtils
+import glob
+import os
+
+# scons steals our environment away, so we have to reinstate it
+env = lsst.sconsUtils.env.Clone()
+for name in ("PYTHONPATH", "LD_LIBRARY_PATH",
+             "DYLD_LIBRARY_PATH", "PATH"):
+    if name in os.environ:
+        env.AppendENVPath(name, os.environ[name])
+
+# we may need an explicit library load path specified in the command
+libpathstr = lsst.sconsUtils.utils.libraryLoaderEnvironment()
+
+# We always run these commands with an explicit python rather than relying on the shebang
+python = "{} python".format(libpathstr)
+
+defectsDatList = glob.glob("defects/*/defects.dat")
+for defectsDat in defectsDatList:
+    command = "%s bin.src/genDefectFits.py auxTel %s --force" % (python, os.path.join("auxTel", defectsDat))
+    commandInst = env.Command(defectsDat + "-fits", [defectsDat], command)
+    env.Depends(commandInst, lsst.sconsUtils.targets["python"])
+    env.AlwaysBuild(commandInst)
+
+command = "%s bin.src/genDefectRegistry.py --root auxTel/defects" % (python)
+commandInst = env.Command("defects/defectRegistry.sqlite3", [d + "-fits" for d in defectsDatList], command)
+env.Depends(commandInst, lsst.sconsUtils.targets["python"])
+registry = env.AlwaysBuild(commandInst)

auxTel/defects/2018-01-01/defects.dat

@@ -0,0 +1,12 @@
+# Defects file
+#
+# Convert to a fits table using genDefectFits.py;  this is done for you by running scons
+#
+# Columns are:
+#  CCD     x0     y0  width height
+#
+# We keep to this format so that it's the same as in other cameras
+# despite only having one CCD
+#
+# Mask for C17 - the bad amp in the lower right
+     0    3563      0     509     2000

bin.src/genDefectFits.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+import sys
+import os.path
+import re
+
+import numpy
+from astropy.io import fits
+import collections
+
+from lsst.obs.lsst.auxTel import AuxTelMapper
+
+Defect = collections.namedtuple('Defect', ['x0', 'y0', 'width', 'height'])
+mapperMap = {'auxTel': AuxTelMapper}
+
+
+def genDefectFits(cameraName, source, targetDir):
+    mapper = mapperMap[cameraName](root=".", calibRoot=".")
+    camera = mapper.camera
+
+    ccds = dict()
+    for ccd in camera:
+        ccdNum = ccd.getId()
+        ccdName = ccd.getName()
+        ccds[ccdNum] = ccdName
+
+    defects = dict()
+
+    with open(source, "r") as f:
+        for line in f:
+            line = re.sub(r"\#.*", "", line).strip()
+            if len(line) == 0:
+                continue
+            ccd, x0, y0, width, height = re.split(r"\s+", line)
+            ccd = int(ccd)
+            if ccd not in ccds:
+                raise RuntimeError("Unrecognised ccd: %d" % ccd)
+            if ccd not in defects:
+                defects[ccd] = list()
+            defects[ccd].append(Defect(x0=int(x0), y0=int(y0), width=int(width), height=int(height)))
+
+    for ccd in ccds:
+        # Make empty defect FITS file for CCDs with no defects
+        if ccd not in defects:
+            defects[ccd] = list()
+
+        columns = list()
+        for colName in Defect._fields:
+            colData = numpy.array([d._asdict()[colName] for d in defects[ccd]])
+            col = fits.Column(name=colName, format="I", array=colData)
+            columns.append(col)
+
+        cols = fits.ColDefs(columns)
+        table = fits.BinTableHDU.from_columns(cols)
+
+        table.header['DETNUM'] = ccd
+        table.header['NAME'] = ccdName
+
+        name = os.path.join(targetDir, "defects_%d.fits" % ccd)
+        print("Writing %d defects from CCD %d (%s) to %s" % (table.header['NAXIS2'], ccd, ccds[ccd], name))
+        if os.path.exists(name):
+            if args.force:
+                os.unlink(name)
+            else:
+                print("File %s already exists; use --force to overwrite" % name, file=sys.stderr)
+                continue
+
+        table.writeto(name)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("cameraName", type=str, choices=['auxTel'],
+                        help="Camera name: auxTel only at present")
+    parser.add_argument("defectsFile", type=str, help="Text file containing list of defects")
+    parser.add_argument("targetDir", type=str, nargs="?", help="Directory for generated fits files")
+    parser.add_argument("-f", "--force", action="store_true", help="Force operations")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Be chattier")
+    args = parser.parse_args()
+
+    if not args.targetDir:
+        args.targetDir = os.path.split(args.defectsFile)[0]
+
+    genDefectFits(args.cameraName, args.defectsFile, args.targetDir)

bin.src/genDefectRegistry.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+import glob
+import os
+import re
+try:
+    import sqlite3 as sqlite
+except ImportError:
+    import sqlite
+import sys
+import argparse
+import datetime
+
+from lsst.pipe.base import Struct
+
+
+class Row(Struct):
+
+    def __init__(self, path, version, validStart, validEnd=None):
+        super(Row, self).__init__(path=path, version=version, validStart=validStart, validEnd=validEnd)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--create", action="store_true", help="Create new registry (clobber old)?")
+parser.add_argument("--root", default=".", help="Root directory")
+parser.add_argument("-v", "--verbose", action="store_true", help="Be chattier")
+args = parser.parse_args()
+
+registryName = os.path.join(args.root, "defectRegistry.sqlite3")
+if args.create and os.path.exists(registryName):
+    os.unlink(registryName)
+
+makeTables = not os.path.exists(registryName)
+
+conn = sqlite.connect(registryName)
+
+if makeTables:
+    cmd = "create table defect (id integer primary key autoincrement"
+    cmd += ", path text, version text, detector int, detectorName text"
+    cmd += ", validStart text, validEnd text)"
+    conn.execute(cmd)
+    conn.commit()
+
+cmd = "INSERT INTO defect VALUES (NULL, ?, ?, ?, ?, ?, ?)"
+
+rowsPerDetector = {}
+for f in glob.glob(os.path.join(args.root, "*", "defects*.fits")):
+    m = re.search(r'(\S+)/defects_(\d+)\.fits', f)
+    if not m:
+        print("Unrecognized file: %s" % (f,), file=sys.stderr)
+        continue
+    #
+    # Convert f to be relative to the location of the registry
+    #
+    f = os.path.relpath(f, args.root)
+
+    startDate = m.group(1).split('/')[-1]
+    version = m.group(1)
+    det = m.group(2)
+    if det not in rowsPerDetector:
+        rowsPerDetector[det] = []
+    rowsPerDetector[det].append(Row(f, version, startDate))
+
+# Fix up end dates so there are no collisions.
+# Defects files for a detector are valid from the date they are registered
+# until the next date. This means that any defects file should carry ALL
+# the defects that are present at that time.
+for det, rowList in rowsPerDetector.items():
+    # ISO-8601 will sort just fine without conversion from str
+    rowList.sort(key=lambda row: row.validStart)
+    for thisRow, nextRow in zip(rowList[:-1], rowList[1:]):
+        thisRow.validEnd = (datetime.datetime.strptime(nextRow.validStart, "%Y-%m-%d") -
+                            datetime.timedelta(0, 1)).isoformat()  # 1 sec before: sqlite precision is 1 sec
+    rowList[-1].validEnd = "2037-12-31"  # End of UNIX time
+
+    for row in rowList:
+        if args.verbose:
+            print("Registering %s" % row.path)
+        conn.execute(cmd, (row.path, row.version, det, "RXX_S00", row.validStart, row.validEnd))
+
+conn.commit()
+conn.close()

config/auxTel/auxTel.py

@@ -27,6 +27,5 @@
     config.ccdKeys = ['detector', 'detectorName']
 
 config.isr.doLinearize = False
-config.isr.doDefect = False
-config.isr.doCrosstalk=False
+config.isr.doCrosstalk = False
 config.isr.doAddDistortionModel = False

config/auxTel/dark.py

@@ -0,0 +1,28 @@
+# This file is part of obs_lsst.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+
+import os.path
+from lsst.utils import getPackageDir
+
+config.load(os.path.join(getPackageDir("obs_lsst"), "config", "auxTel", "auxTel.py"))
+
+config.repair.cosmicray.nCrPixelMax = 100000

config/auxTel/flat.py

@@ -0,0 +1,26 @@
+# This file is part of obs_lsst.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+
+import os.path
+from lsst.utils import getPackageDir
+
+config.load(os.path.join(getPackageDir("obs_lsst"), "config", "auxTel", "auxTel.py"))

config/auxTel/fringe.py

@@ -0,0 +1,26 @@
+# This file is part of obs_lsst.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+
+import os.path
+from lsst.utils import getPackageDir
+
+config.load(os.path.join(getPackageDir("obs_lsst"), "config", "auxTel", "auxTel.py"))

policy/auxTel/auxTelMapper.yaml

@@ -1,3 +1,5 @@
+defects: ../auxTel/defects
+
 calibrations:
   bias:
     obsTimeName: dayObs

python/lsst/obs/lsst/auxTel.py

@@ -77,6 +77,12 @@ def _computeCcdExposureId(self, dataId):
 
         return LsstAuxTelTranslator.compute_detector_exposure_id(visit, detector)
 
+    def _getCcdKeyVal(self, dataId):
+        """Return CCD key and value used to look up a defect in the defect
+        registry.
+        """
+        return ("detector", 0)
+
 
 class AuxTelParseTask(LsstCamParseTask):
     """Parser suitable for auxTel data.

python/lsst/obs/lsst/lsstCamMapper.py

@@ -339,6 +339,14 @@ def _extractDetectorName(self, dataId):
 
         return "%s_%s" % (raftName, detectorName)
 
+    def _defectLookup(self, dataId):
+        """Find the defects for a given CCD.
+
+        This method just exists to supplement the dateKey, as we don't have
+        the default, taiObs, in the obs_lsst registry.
+        """
+        return super()._defectLookup(dataId, dateKey='dayObs')
+
     def _computeCcdExposureId(self, dataId):
         """Compute the 64-bit (long) identifier for a CCD exposure.
 

python/lsst/obs/lsst/translators/auxTel.py

@@ -181,7 +181,7 @@ def to_observation_type(self):
         obstype : `str`
             Observation type.
         """
-        if self._is_on_mountain():
+        if self._is_on_mountain() or "IMGTYPE" in self._header:
             obstype = self._header["IMGTYPE"]
             self._used_these_cards("IMGTYPE")
             return obstype.lower()