Skip to content

Commit

Permalink
Merge pull request #30 from lsst/tickets/DM-23699
Browse files Browse the repository at this point in the history
DM-23699: Update fgcmcal default config format
  • Loading branch information
erykoff committed Mar 10, 2020
2 parents 2fc2e58 + 11b7162 commit 2c93450
Show file tree
Hide file tree
Showing 7 changed files with 383 additions and 82 deletions.
10 changes: 9 additions & 1 deletion python/lsst/fgcmcal/fgcmCalibrateTract.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,21 @@ def setDefaults(self):

self.fgcmBuildStars.checkAllCcds = False
self.fgcmBuildStars.densityCutMaxPerPixel = 10000
self.fgcmFitCycle.useRepeatabilityForExpGrayCuts = [True]
self.fgcmFitCycle.quietMode = True
self.fgcmOutputProducts.doReferenceCalibration = False
self.fgcmOutputProducts.doRefcatOutput = False
self.fgcmOutputProducts.cycleNumber = 0
self.fgcmOutputProducts.photoCal.applyColorTerms = False

def validate(self):
super().validate()

for band in self.fgcmFitCycle.bands:
if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands'
raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
self, msg)


class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner):
"""Subclass of TaskRunner for FgcmCalibrateTractTask
Expand Down
266 changes: 224 additions & 42 deletions python/lsst/fgcmcal/fgcmFitCycle.py

Large diffs are not rendered by default.

28 changes: 22 additions & 6 deletions python/lsst/fgcmcal/fgcmOutputProducts.py
Original file line number Diff line number Diff line change
Expand Up @@ -761,14 +761,30 @@ def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None):

self.log.info("Outputting %s objects" % (datasetType))

# Only output those that we have a calibration
# See fgcmFitCycle._makeZptSchema for flag definitions
selected = (zptCat['fgcmFlag'] < 16)

# Get the mapping from filtername to dataId filter name
# Select visit/ccds where we have a calibration
# This includes ccds where we were able to interpolate from neighboring
# ccds.
cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT']
too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD']
selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) &
(zptCat['fgcmZptVar'] > 0.0))

# We also select the "best" calibrations, avoiding interpolation. These
# are only used for mapping filternames
selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) &
(zptCat['fgcmZptVar'] > 0.0))

# Log warnings for any visit which has no valid zeropoints
badVisits = np.unique(zptCat['visit'][~selected])
goodVisits = np.unique(zptCat['visit'][selected])
allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
for allBadVisit in allBadVisits:
self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}')

# Get the mapping from filtername to dataId filter name, empirically
filterMapping = {}
nFound = 0
for rec in zptCat[selected]:
for rec in zptCat[selected_best]:
if rec['filtername'] in filterMapping:
continue
dataId = {self.visitDataRefName: int(rec['visit']),
Expand Down
35 changes: 15 additions & 20 deletions python/lsst/fgcmcal/utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,8 @@ def makeConfigDict(config, log, camera, maxIter,
configDict: `dict`
Configuration dictionary for fgcm
"""

fitFlag = np.array(config.fitFlag, dtype=np.bool)
requiredFlag = np.array(config.requiredFlag, dtype=np.bool)

fitBands = [b for i, b in enumerate(config.bands) if fitFlag[i]]
notFitBands = [b for i, b in enumerate(config.bands) if not fitFlag[i]]
requiredBands = [b for i, b in enumerate(config.bands) if requiredFlag[i]]
# Extract the bands that are _not_ being fit for fgcm configuration
notFitBands = [b for b in config.bands if b not in config.fitBands]

# process the starColorCuts
starColorCutList = []
Expand Down Expand Up @@ -113,9 +108,9 @@ def makeConfigDict(config, log, camera, maxIter,
'skyBrightnessField': 'SKYBACKGROUND',
'deepFlag': 'DEEPFLAG', # unused
'bands': list(config.bands),
'fitBands': list(fitBands),
'notFitBands': list(notFitBands),
'requiredBands': list(requiredBands),
'fitBands': list(config.fitBands),
'notFitBands': notFitBands,
'requiredBands': list(config.requiredBands),
'filterToBand': dict(config.filterMap),
'logLevel': 'INFO', # FIXME
'nCore': config.nCore,
Expand All @@ -124,11 +119,11 @@ def makeConfigDict(config, log, camera, maxIter,
'reserveFraction': config.reserveFraction,
'freezeStdAtmosphere': config.freezeStdAtmosphere,
'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle,
'superStarSubCCD': config.superStarSubCcd,
'superStarSubCCDDict': dict(config.superStarSubCcdDict),
'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder,
'superStarSubCCDTriangular': config.superStarSubCcdTriangular,
'superStarSigmaClip': config.superStarSigmaClip,
'ccdGraySubCCD': config.ccdGraySubCcd,
'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict),
'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder,
'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular,
'cycleNumber': config.cycleNumber,
Expand All @@ -146,25 +141,25 @@ def makeConfigDict(config, log, camera, maxIter,
'minStarPerExp': config.minStarPerExp,
'minExpPerNight': config.minExpPerNight,
'expGrayInitialCut': config.expGrayInitialCut,
'expGrayPhotometricCut': np.array(config.expGrayPhotometricCut),
'expGrayHighCut': np.array(config.expGrayHighCut),
'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict),
'expGrayHighCutDict': dict(config.expGrayHighCutDict),
'expGrayRecoverCut': config.expGrayRecoverCut,
'expVarGrayPhotometricCut': config.expVarGrayPhotometricCut,
'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict),
'expGrayErrRecoverCut': config.expGrayErrRecoverCut,
'refStarSnMin': config.refStarSnMin,
'refStarOutlierNSig': config.refStarOutlierNSig,
'applyRefStarColorCuts': config.applyRefStarColorCuts,
'illegalValue': -9999.0, # internally used by fgcm.
'starColorCuts': starColorCutList,
'aperCorrFitNBins': config.aperCorrFitNBins,
'aperCorrInputSlopes': np.array(config.aperCorrInputSlopes),
'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict),
'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'],
'sedTermDict': config.sedterms.toDict()['data'],
'colorSplitIndices': np.array(config.colorSplitIndices),
'colorSplitBands': list(config.colorSplitBands),
'sigFgcmMaxErr': config.sigFgcmMaxErr,
'sigFgcmMaxEGray': list(config.sigFgcmMaxEGray),
'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict),
'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr,
'approxThroughput': list(config.approxThroughput),
'approxThroughputDict': dict(config.approxThroughputDict),
'sigmaCalRange': list(config.sigmaCalRange),
'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile),
'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile),
Expand All @@ -183,7 +178,7 @@ def makeConfigDict(config, log, camera, maxIter,
'instrumentParsPerBand': config.instrumentParsPerBand,
'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT,
'fitMirrorChromaticity': config.fitMirrorChromaticity,
'useRepeatabilityForExpGrayCuts': list(config.useRepeatabilityForExpGrayCuts),
'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict),
'autoPhotometricCutNSig': config.autoPhotometricCutNSig,
'autoHighCutNSig': config.autoHighCutNSig,
'printOnly': False,
Expand Down
25 changes: 15 additions & 10 deletions tests/config/fgcmFitCycleHsc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,23 @@
config.outfileBase = 'TestFgcm'
config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i'}
config.bands = ['g', 'r', 'i']
config.fitFlag = (1, 1, 1)
config.requiredFlag = (0, 1, 1)
config.fitBands = ['g', 'r', 'i']
config.requiredBands = ['r', 'i']
config.maxIterBeforeFinalCycle = 5
config.nCore = 1
config.washMjds = (0.0, )
config.epochMjds = (0.0, 100000.0)
config.expGrayPhotometricCut = (-0.1, -0.1, -0.1)
config.expGrayHighCut = (0.1, 0.1, 0.1)
config.expVarGrayPhotometricCut = 0.05**2.
config.expGrayPhotometricCutDict = {'g': -0.1, 'r': -0.1, 'i': -0.1}
config.expGrayHighCutDict = {'g': 0.1, 'r': 0.1, 'i': 0.1}
config.expVarGrayPhotometricCutDict = {'g': 0.05**2.,
'r': 0.05**2.,
'i': 0.05**2.}
config.autoPhotometricCutNSig = 5.0
config.autoHighCutNSig = 5.0
config.aperCorrFitNBins = 2
config.aperCorrInputSlopes = (-1.0, -0.9694, -1.7229)
config.aperCorrInputSlopeDict = {'g': -1.0,
'r': -0.9694,
'i': -1.7229}
config.sedboundaryterms = fgcmcal.SedboundarytermDict()
config.sedboundaryterms.data['ri'] = fgcmcal.Sedboundaryterm(primary='r',
secondary='i')
Expand All @@ -32,11 +36,12 @@
config.minStarPerExp = 50
config.nStarPerRun = 50
config.nExpPerRun = 2
config.colorSplitIndices = (1, 2)
config.colorSplitBands = ['r', 'i']
config.superStarSubCcdChebyshevOrder = 1
config.ccdGraySubCcd = False
config.ccdGraySubCcdDict = {'g': False,
'r': False,
'i': False}
config.modelMagErrors = False
config.sigmaCalRange = (0.003, 0.003)
config.instrumentParsPerBand = False
config.useRepeatabilityForExpGrayCuts = (False, False, False)
config.sigFgcmMaxEGray = (0.05, 0.05, 0.05)

6 changes: 3 additions & 3 deletions tests/test_fgcmcal_hsc.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ def test_fgcmcalTasks(self):
# Test the "final" fit cycle
newConfig = copy.copy(self.config)
newConfig.update(cycleNumber=2,
ccdGraySubCcd=True,
ccdGraySubCcdDict={'g': True, 'r': True, 'i': True},
isFinalCycle=True)
self.config = newConfig

Expand Down Expand Up @@ -177,8 +177,8 @@ def test_fgcmcalTract(self):
self.otherArgs = []

rawRepeatability = np.array([0.0, 0.00691888829016613, 0.00443888382172])
filterNCalibMap = {'HSC-R': 13,
'HSC-I': 14}
filterNCalibMap = {'HSC-R': 17,
'HSC-I': 16}

visits = [903334, 903336, 903338, 903342, 903344, 903346,
903986, 903988, 903990, 904010, 904014]
Expand Down
95 changes: 95 additions & 0 deletions tests/test_fitcycle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# See COPYRIGHT file at the top of the source tree.
#
# This file is part of fgcmcal.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (https://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Test the fitcycle configurations
"""

import unittest
import copy

import lsst.fgcmcal as fgcmcal
import lsst.utils


class FgcmcalTestFitCycleConfig(lsst.utils.tests.TestCase):
"""
Test FgcmFitCycleConfig validation.
"""
def test_fgcmfitcycle_config_validation(self):
"""
Test the FgcmFitCycleConfig validation.
"""
config = fgcmcal.FgcmFitCycleConfig()

config.cycleNumber = 0
config.utBoundary = 0.0
config.latitude = 0.0
config.outfileBase = 'None'
config.bands = ['r', 'i']
config.fitBands = ['r', 'i']
config.requiredBands = ['r']
config.colorSplitBands = ['r', 'i']
config.superStarSubCcdDict = {'r': True, 'i': True}
config.ccdGraySubCcdDict = {'r': True, 'i': True}
config.expGrayPhotometricCutDict = {'r': -0.05, 'i': -0.05}
config.expGrayHighCutDict = {'r': 0.10, 'i': 0.10}
config.expVarGrayPhotometricCutDict = {'r': 0.0005, 'i': 0.0005}
config.sigFgcmMaxEGrayDict = {'r': 0.05, 'i': 0.05}
config.approxThroughputDict = {'r': 1.0, 'i': 1.0}
config.useRepeatabilityForExpGrayCutsDict = {'r': False, 'i': False}

# Ensure that it validates
config.validate()

self._test_misconfig(config, 'fitBands', ['r', 'i', 'z'])
self._test_misconfig(config, 'requiredBands', ['r', 'i', 'z'])
self._test_misconfig(config, 'colorSplitBands', ['r', 'z'])
self._test_misconfig(config, 'superStarSubCcdDict', {'r': True})
self._test_misconfig(config, 'ccdGraySubCcdDict', {'r': True})
self._test_misconfig(config, 'expGrayPhotometricCutDict', {'r': -0.05})
self._test_misconfig(config, 'expGrayHighCutDict', {'r': 0.10})
self._test_misconfig(config, 'expVarGrayPhotometricCutDict', {'r': 0.0005})
self._test_misconfig(config, 'sigFgcmMaxEGrayDict', {'r': 0.05})
self._test_misconfig(config, 'approxThroughputDict', {'r': 1.0})
self._test_misconfig(config, 'useRepeatabilityForExpGrayCutsDict', {'r': False})

def _test_misconfig(self, config, field, value):
"""
Test misconfigured field.
"""
config2 = copy.copy(config)
config2.update(**{field: value})
with self.assertRaises(ValueError):
config2.validate()


class TestMemory(lsst.utils.tests.MemoryTestCase):
pass


def setup_module(module):
lsst.utils.tests.init()


if __name__ == "__main__":
lsst.utils.tests.init()
unittest.main()

0 comments on commit 2c93450

Please sign in to comment.