Merge pull request #272 from lsst/tickets/DM-39639

DM-39639: Add ability to hold auxiliary per-detector per-exposure data.

python/lsst/ip/isr/ptcDataset.py

@@ -160,18 +160,22 @@ class PhotonTransferCurveDataset(IsrCalib):
         will be right-padded with np.nan to match the length of
         rawExpTimes.
     photoCharges : `dict`, [`str`, `np.ndarray`]
-         Dictionary keyed by amp names containing the integrated photocharge
-         for linearity calibration.
+        Dictionary keyed by amp names containing the integrated photocharge
+        for linearity calibration.
+    auxValues : `dict`, [`str`, `np.ndarray`]
+        Dictionary of per-detector auxiliary header values that can be used
+        for PTC, linearity computation.
 
     Version 1.1 adds the `ptcTurnoff` attribute.
     Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues`
     attributes.
     Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes.
+    Version 1.4 adds the `auxValues` attribute.
     """
 
     _OBSTYPE = 'PTC'
     _SCHEMA = 'Gen3 Photon Transfer Curve'
-    _VERSION = 1.3
+    _VERSION = 1.4
 
     def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
         self.ptcFitType = ptcFitType
@@ -215,6 +219,9 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
         self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
         self.finalMeans = {ampName: np.array([]) for ampName in ampNames}
 
+        # Try this as a dict of arrays.
+        self.auxValues = {}
+
         super().__init__(**kwargs)
         self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
                                         'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
@@ -223,7 +230,7 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
                                         'covariancesSqrtWeights', 'covariancesModelNoB',
                                         'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
                                         'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
-                                        'histChi2Dofs', 'kspValues'])
+                                        'histChi2Dofs', 'kspValues', 'auxValues'])
 
         self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)
 
@@ -243,6 +250,7 @@ def setAmpValuesPartialDataset(
             histVar=np.nan,
             histChi2Dof=np.nan,
             kspValue=0.0,
+            auxValues=None,
     ):
         """
         Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
@@ -307,6 +315,18 @@ def setAmpValuesPartialDataset(
         self.noiseMatrix[ampName] = nanMatrix
         self.noiseMatrixNoB[ampName] = nanMatrix
 
+    def setAuxValuesPartialDataset(self, auxDict):
+        """
+        Set a dictionary of auxiliary values for a partial dataset.
+
+        Parameters
+        ----------
+        auxDict : `dict` [`str`, `float`]
+            Dictionary of float values.
+        """
+        for key, value in auxDict.items():
+            self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
+
     def updateMetadata(self, **kwargs):
         """Update calibration metadata.
         This calls the base class's method after ensuring the required
@@ -426,6 +446,9 @@ def fromDict(cls, dictionary):
             calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
             calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)
 
+        for key, value in dictionary['auxValues'].items():
+            calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
+
         calib.updateMetadata()
         return calib
 
@@ -485,6 +508,7 @@ def _dictOfArraysToDictOfLists(dictOfArrays):
         outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
         outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
         outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
+        outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues)
 
         return outDict
 
@@ -605,6 +629,13 @@ def fromTable(cls, tableList):
                 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
                 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']
 
+        inDict['auxValues'] = {}
+        record = ptcTable[0]
+        for col in record.columns.keys():
+            if col.startswith('PTCAUX_'):
+                parts = col.split('PTCAUX_')
+                inDict['auxValues'][parts[1]] = record[col]
+
         return cls().fromDict(inDict)
 
     def toTable(self):
@@ -662,6 +693,11 @@ def toTable(self):
                 'FINAL_MODEL_VARS': self.finalModelVars[ampName],
                 'FINAL_MEANS': self.finalMeans[ampName],
             }
+
+            if self.auxValues:
+                for key, value in self.auxValues.items():
+                    ampDict[f"PTCAUX_{key}"] = value
+
             catalogList.append(ampDict)
 
         catalog = Table(catalogList)

tests/test_ptcDataset.py

@@ -129,6 +129,10 @@ def _checkTypes(self, ptcDataset):
             self.assertIsInstance(ptcDataset.photoCharges[ampName], np.ndarray)
             self.assertEqual(ptcDataset.photoCharges[ampName].dtype, np.float64)
 
+        for key, value in ptcDataset.auxValues.items():
+            self.assertIsInstance(value, np.ndarray)
+            self.assertEqual(value.dtype, np.float64)
+
     def test_emptyPtcDataset(self):
         """Test an empty PTC dataset."""
         emptyDataset = PhotonTransferCurveDataset(
@@ -169,19 +173,28 @@ def test_partialPtcDataset(self):
                 rawMean=10.0,
                 rawVar=10.0,
             )
-        self._checkTypes(partialDataset)
 
-        with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
-            usedFilename = partialDataset.writeText(f.name)
-            fromText = PhotonTransferCurveDataset.readText(usedFilename)
-        self.assertEqual(fromText, partialDataset)
-        self._checkTypes(fromText)
+        for useAuxValues in [False, True]:
+            if useAuxValues:
+                partialDataset.setAuxValuesPartialDataset(
+                    {
+                        "CCOBCURR": 1.0,
+                        "CCDTEMP": 0.0,
+                    }
+                )
+            self._checkTypes(partialDataset)
 
-        with tempfile.NamedTemporaryFile(suffix=".fits") as f:
-            usedFilename = partialDataset.writeFits(f.name)
-            fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
-        self.assertEqual(fromFits, partialDataset)
-        self._checkTypes(fromFits)
+            with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
+                usedFilename = partialDataset.writeText(f.name)
+                fromText = PhotonTransferCurveDataset.readText(usedFilename)
+            self.assertEqual(fromText, partialDataset)
+            self._checkTypes(fromText)
+
+            with tempfile.NamedTemporaryFile(suffix=".fits") as f:
+                usedFilename = partialDataset.writeFits(f.name)
+                fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
+            self.assertEqual(fromFits, partialDataset)
+            self._checkTypes(fromFits)
 
     def test_ptcDatset(self):
         """Test of a full PTC dataset."""
@@ -260,19 +273,26 @@ def test_ptcDatset(self):
                     localDataset.noiseMatrixNoB[ampName] = np.full(
                         (nSideCovMatrix, nSideCovMatrix), 3.0)
 
-            self._checkTypes(localDataset)
-
-            with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
-                usedFilename = localDataset.writeText(f.name)
-                fromText = PhotonTransferCurveDataset.readText(usedFilename)
-            self.assertEqual(fromText, localDataset)
-            self._checkTypes(fromText)
-
-            with tempfile.NamedTemporaryFile(suffix=".fits") as f:
-                usedFilename = localDataset.writeFits(f.name)
-                fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
-            self.assertEqual(fromFits, localDataset)
-            self._checkTypes(fromFits)
+            for useAuxValues in [False, True]:
+                if useAuxValues:
+                    localDataset.auxValues = {
+                        "CCOBCURR": np.ones(nSignalPoints),
+                        "CCDTEMP": np.zeros(nSignalPoints),
+                    }
+
+                self._checkTypes(localDataset)
+
+                with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
+                    usedFilename = localDataset.writeText(f.name)
+                    fromText = PhotonTransferCurveDataset.readText(usedFilename)
+                self.assertEqual(fromText, localDataset)
+                self._checkTypes(fromText)
+
+                with tempfile.NamedTemporaryFile(suffix=".fits") as f:
+                    usedFilename = localDataset.writeFits(f.name)
+                    fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
+                self.assertEqual(fromFits, localDataset)
+                self._checkTypes(fromFits)
 
     def test_getExpIdsUsed(self):
         localDataset = copy.copy(self.dataset)