Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-39639: Add ability to hold auxiliary per-detector per-exposure data. #272

Merged
merged 1 commit into from
Jun 26, 2023
Merged

DM-39639: Add ability to hold auxiliary per-detector per-exposure data. #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
44 changes: 40 additions & 4 deletions python/lsst/ip/isr/ptcDataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,18 +160,22 @@ class PhotonTransferCurveDataset(IsrCalib):
will be right-padded with np.nan to match the length of
rawExpTimes.
photoCharges : `dict`, [`str`, `np.ndarray`]
Dictionary keyed by amp names containing the integrated photocharge
for linearity calibration.
Dictionary keyed by amp names containing the integrated photocharge
for linearity calibration.
auxValues : `dict`, [`str`, `np.ndarray`]
Dictionary of per-detector auxiliary header values that can be used
for PTC, linearity computation.

Version 1.1 adds the `ptcTurnoff` attribute.
Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues`
attributes.
Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes.
Version 1.4 adds the `auxValues` attribute.
"""

_OBSTYPE = 'PTC'
_SCHEMA = 'Gen3 Photon Transfer Curve'
_VERSION = 1.3
_VERSION = 1.4

def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
self.ptcFitType = ptcFitType
Expand Down Expand Up @@ -215,6 +219,9 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
self.finalMeans = {ampName: np.array([]) for ampName in ampNames}

# Try this as a dict of arrays.
self.auxValues = {}

super().__init__(**kwargs)
self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
Expand All @@ -223,7 +230,7 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
'covariancesSqrtWeights', 'covariancesModelNoB',
'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
'histChi2Dofs', 'kspValues'])
'histChi2Dofs', 'kspValues', 'auxValues'])

self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)

Expand All @@ -243,6 +250,7 @@ def setAmpValuesPartialDataset(
histVar=np.nan,
histChi2Dof=np.nan,
kspValue=0.0,
auxValues=None,
):
"""
Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
Expand Down Expand Up @@ -307,6 +315,18 @@ def setAmpValuesPartialDataset(
self.noiseMatrix[ampName] = nanMatrix
self.noiseMatrixNoB[ampName] = nanMatrix

def setAuxValuesPartialDataset(self, auxDict):
"""
Set a dictionary of auxiliary values for a partial dataset.

Parameters
----------
auxDict : `dict` [`str`, `float`]
Dictionary of float values.
"""
for key, value in auxDict.items():
self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))

def updateMetadata(self, **kwargs):
"""Update calibration metadata.
This calls the base class's method after ensuring the required
Expand Down Expand Up @@ -426,6 +446,9 @@ def fromDict(cls, dictionary):
calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)

for key, value in dictionary['auxValues'].items():
calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))

calib.updateMetadata()
return calib

Expand Down Expand Up @@ -485,6 +508,7 @@ def _dictOfArraysToDictOfLists(dictOfArrays):
outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues)

return outDict

Expand Down Expand Up @@ -605,6 +629,13 @@ def fromTable(cls, tableList):
inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']

inDict['auxValues'] = {}
record = ptcTable[0]
for col in record.columns.keys():
if col.startswith('PTCAUX_'):
parts = col.split('PTCAUX_')
inDict['auxValues'][parts[1]] = record[col]

return cls().fromDict(inDict)

def toTable(self):
Expand Down Expand Up @@ -662,6 +693,11 @@ def toTable(self):
'FINAL_MODEL_VARS': self.finalModelVars[ampName],
'FINAL_MEANS': self.finalMeans[ampName],
}

if self.auxValues:
for key, value in self.auxValues.items():
ampDict[f"PTCAUX_{key}"] = value

catalogList.append(ampDict)

catalog = Table(catalogList)
Expand Down
68 changes: 44 additions & 24 deletions tests/test_ptcDataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ def _checkTypes(self, ptcDataset):
self.assertIsInstance(ptcDataset.photoCharges[ampName], np.ndarray)
self.assertEqual(ptcDataset.photoCharges[ampName].dtype, np.float64)

for key, value in ptcDataset.auxValues.items():
self.assertIsInstance(value, np.ndarray)
self.assertEqual(value.dtype, np.float64)

def test_emptyPtcDataset(self):
"""Test an empty PTC dataset."""
emptyDataset = PhotonTransferCurveDataset(
Expand Down Expand Up @@ -169,19 +173,28 @@ def test_partialPtcDataset(self):
rawMean=10.0,
rawVar=10.0,
)
self._checkTypes(partialDataset)

with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
usedFilename = partialDataset.writeText(f.name)
fromText = PhotonTransferCurveDataset.readText(usedFilename)
self.assertEqual(fromText, partialDataset)
self._checkTypes(fromText)
for useAuxValues in [False, True]:
if useAuxValues:
partialDataset.setAuxValuesPartialDataset(
{
"CCOBCURR": 1.0,
"CCDTEMP": 0.0,
}
)
self._checkTypes(partialDataset)

with tempfile.NamedTemporaryFile(suffix=".fits") as f:
usedFilename = partialDataset.writeFits(f.name)
fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
self.assertEqual(fromFits, partialDataset)
self._checkTypes(fromFits)
with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
usedFilename = partialDataset.writeText(f.name)
fromText = PhotonTransferCurveDataset.readText(usedFilename)
self.assertEqual(fromText, partialDataset)
self._checkTypes(fromText)

with tempfile.NamedTemporaryFile(suffix=".fits") as f:
usedFilename = partialDataset.writeFits(f.name)
fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
self.assertEqual(fromFits, partialDataset)
self._checkTypes(fromFits)

def test_ptcDatset(self):
"""Test of a full PTC dataset."""
Expand Down Expand Up @@ -260,19 +273,26 @@ def test_ptcDatset(self):
localDataset.noiseMatrixNoB[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), 3.0)

self._checkTypes(localDataset)

with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
usedFilename = localDataset.writeText(f.name)
fromText = PhotonTransferCurveDataset.readText(usedFilename)
self.assertEqual(fromText, localDataset)
self._checkTypes(fromText)

with tempfile.NamedTemporaryFile(suffix=".fits") as f:
usedFilename = localDataset.writeFits(f.name)
fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
self.assertEqual(fromFits, localDataset)
self._checkTypes(fromFits)
for useAuxValues in [False, True]:
if useAuxValues:
localDataset.auxValues = {
"CCOBCURR": np.ones(nSignalPoints),
"CCDTEMP": np.zeros(nSignalPoints),
}

self._checkTypes(localDataset)

with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
usedFilename = localDataset.writeText(f.name)
fromText = PhotonTransferCurveDataset.readText(usedFilename)
self.assertEqual(fromText, localDataset)
self._checkTypes(fromText)

with tempfile.NamedTemporaryFile(suffix=".fits") as f:
usedFilename = localDataset.writeFits(f.name)
fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
self.assertEqual(fromFits, localDataset)
self._checkTypes(fromFits)

def test_getExpIdsUsed(self):
localDataset = copy.copy(self.dataset)
Expand Down