DM-18739: Generate defects from standardized format

SConstruct

@@ -1,3 +1,3 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
-scripts.BasicSConstruct("obs_test")
+scripts.BasicSConstruct("obs_test", defaultTargets=scripts.DEFAULT_TARGETS + ("data",))

config/ingestCalibs.py

@@ -10,6 +10,6 @@
                            }
 config.register.detector = ['filter']
 config.register.visit = ['calibDate']
-config.register.tables = ['bias', 'flat']  # defects are hardcoded into mapper; cannot be ingested
+config.register.tables = ['bias', 'flat', 'defects']
 config.register.validityUntilSuperseded = config.register.tables
 config.register.unique = ['filter', 'calibDate']

config/ingestDefects.py

@@ -0,0 +1,14 @@
+# Config override for lsst.pipe.tasks.IngestCalibsTask
+config.parse.translation = {'calibDate': 'CALIBDATE',
+                            'filter': 'FILTER',
+                            'ccd': 'DETECTOR',
+                           }
+config.register.columns = {'calibDate': 'text',
+                           'validStart': 'text',
+                           'validEnd': 'text',
+                           'filter': 'text',
+                           'ccd': 'int',
+                           }
+config.register.visit = ['calibDate']
+config.register.validityUntilSuperseded = ['defects',]
+config.register.unique = ['calibDate']

data/ReadMe.md

@@ -55,7 +55,11 @@ To make an obs_test defects file from the bias frame generated above, from the o
 
     setup -r .
     data/utils/defectsFromBias.py data/input/bias/bias.fits.gz
-    mv defects_c0.fits data/input/defects/
+    mv defects_c0.ecsv $OBS_TEST_DATA_DIR/test/defects/0/19700101T000000.ecsv
+
+The defects are persisted in the ingested for because ingestion depends on pipe_tasks which introduces a circular dependency.
+
+    ingestDefects.py data/input $OBS_TEST_DATA_DIR/test/defects --calib data/inpu
 
 ## Hints for using the data
 

data/utils/defectsFromBias.py

@@ -21,86 +21,20 @@
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 import argparse
-import time
 
-import numpy
-from astropy.io import fits
+from dateutil import parser as date_parser
 
-import lsst.afw.geom as afwGeom
 import lsst.afw.image as afwImage
-from lsst.ip.isr import getDefectListFromMask
+from lsst.meas.algorithms import Defects
 
-DefectsPath = "defects_c0.fits"
+DefectsPath = "defects_c0"
 """Output path for defects file."""
 detectorName = "0"
 """Detector name."""
 detectorSerial = "0000011"
 """Detector serial code"""
 
 
-def getBBoxList(path, detectorName):
-    """Read a defects file and return the defects as a list of bounding
-    boxes.
-
-    Parameters
-    ---------
-    path : `str`
-        Path to input defects file; a fits file.
-    detectorName : `str`
-        Name of detector.
-    """
-    with fits.open(path) as hduList:
-        for hdu in hduList[1:]:
-            if hdu.header["name"] != detectorName:
-                print("skipping header with name=%r" % (hdu.header["name"],))
-                continue
-
-            bboxList = []
-            for data in hdu.data:
-                bbox = afwGeom.Box2I(
-                    afwGeom.Point2I(int(data['x0']), int(data['y0'])),
-                    afwGeom.Extent2I(int(data['width']), int(data['height'])),
-                )
-                bboxList.append(bbox)
-            return bboxList
-        raise RuntimeError("Data not found for detector %r" % (detectorName,))
-
-
-def writeDefectsFile(bboxList, path, detectorSerial, detectorName):
-    """Write a defects FITS file.
-
-    Parameters
-    ----------
-    bboxList : `list` of `lsst.geom.Box2I`
-        List of bounding boxes defining defect locations.
-    path : `str`
-        Path of output defects file; should end with ".fits".
-    detectorSerial : `str`
-        Serial code of detector.
-    detectorName : `str`
-        Name of detector.
-    """
-    head = fits.Header()
-    head.update('SERIAL', detectorSerial, 'Serial of the detector')
-    head.update('NAME', detectorName, 'Name of detector for this defect map')
-    head.update('CDATE', time.asctime(time.gmtime()), 'UTC of creation')
-
-    x0 = numpy.array([d.getMinX() for d in bboxList])
-    y0 = numpy.array([d.getMinY() for d in bboxList])
-    width = numpy.array([d.getWidth() for d in bboxList])
-    height = numpy.array([d.getHeight() for d in bboxList])
-
-    col1 = fits.Column(name='x0', format='I', array=numpy.array(x0))
-    col2 = fits.Column(name='y0', format='I', array=numpy.array(y0))
-    col3 = fits.Column(name='height', format='I', array=numpy.array(height))
-    col4 = fits.Column(name='width', format='I', array=numpy.array(width))
-    cols = fits.ColDefs([col1, col2, col3, col4])
-    tbhdu = fits.new_table(cols, header=head)
-    hdu = fits.PrimaryHDU()
-    thdulist = fits.HDUList([hdu, tbhdu])
-    thdulist.writeto(DefectsPath)
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="""Construct a defects file from the mask plane of a test camera bias frame.
@@ -112,13 +46,20 @@ def writeDefectsFile(bboxList, path, detectorSerial, detectorName):
     args = parser.parse_args()
 
     biasMI = afwImage.MaskedImageF(args.bias)
-    defectList = getDefectListFromMask(biasMI, "BAD")
-    bboxList = [defect.getBBox() for defect in defectList]
-    writeDefectsFile(bboxList, DefectsPath, detectorSerial, detectorName)
+    defectList = Defects.fromMask(biasMI, "BAD")
+    valid_start = date_parser.parse('19700101T000000')
+    md = defectList.getMetadata()
+    md['INSTRUME'] = 'test'
+    md['DETECTOR'] = detectorName
+    md['CALIBDATE'] = valid_start.isoformat()
+    md['FILTER'] = None
+    md['CALIB_ID'] = (f'detector={detectorName} calibDate={valid_start.isoformat()} '
+                      'ccd={detectorName} ccdnum={detectorName} filter=None')
+    defectList.writeText(DefectsPath)
     print("wrote defects file %r" % (DefectsPath,))
 
-    test2BBoxList = getBBoxList(DefectsPath, detectorName)
-    assert len(bboxList) == len(test2BBoxList)
-    for boxA, boxB in zip(bboxList, test2BBoxList):
-        assert boxA == boxB
+    test2defectList = Defects.readText(DefectsPath+".ecsv")
+    assert len(defectList) == len(test2defectList)
+    for dA, dB in zip(defectList, test2defectList):
+        assert dA.getBBox() == dB.getBBox()
     print("verified that defects file %r round trips correctly" % (DefectsPath,))

policy/testMapper.yaml

@@ -1,5 +1,3 @@
-defects: ../description/defects
-
 needCalibRegistry: false
 
 levels:
@@ -48,6 +46,13 @@ exposures:
     template: fcr/v%(visit)d_f%(filter)s.fcr.fits
 
 calibrations:
+  defects:
+    level: Ccd
+    persistable: BaseCatalog
+    python: lsst.meas.algorithms.Defects
+    storage: FitsCatalogStorage
+    tables: defects
+    template: defects/defects.fits
   bias:
     level: Ccd
     persistable: ExposureF

ups/obs_test.table

@@ -5,6 +5,7 @@ setupRequired(pex_config)
 setupRequired(pex_policy)
 setupRequired(numpy)
 setupRequired(utils)
+setupRequired(obs_test_data)
 setupOptional(astropy)
 
 envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python)