Update cookbook configurations

cookbook/fgcmBuildStarsHsc.py

@@ -18,9 +18,6 @@
 config.densityCutNside = 128
 # If there are more than densityCutMaxPerPixel stars per pixel, sample them
 config.densityCutMaxPerPixel = 1500
-# Dictionary that maps "filters" (instrumental configurations) to "bands"
-# (abstract names).  All filters must be listed in the LUT.
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
 # Which bands are required to be observed to be considered a calibration star
 config.requiredBands = ['g', 'r', 'i', 'z']
 # The reference CCD is a good CCD used to select visit to speed up the scanning

cookbook/fgcmBuildStarsTableHsc.py

@@ -16,9 +16,6 @@
 config.densityCutNside = 128
 # If there are more than densityCutMaxPerPixel stars per pixel, sample them
 config.densityCutMaxPerPixel = 1500
-# Dictionary that maps "filters" (instrumental configurations) to "bands"
-# (abstract names).  All filters must be listed in the LUT.
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
 # Which bands are required to be observed to be considered a calibration star
 config.requiredBands = ['g', 'r', 'i', 'z']
 # The reference CCD is a good CCD used to select visit to speed up the scanning

cookbook/fgcmFitCycleHscCookbook_cycle00_config.py

@@ -12,7 +12,7 @@
 config.fitBands = ('g', 'r', 'i', 'z', 'y')
 # Bands that are required for a star to be considered a calibration star.
 config.requiredBands = ('g', 'r', 'i', 'z', 'y')
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
+
 
 # Maximum number of fit iterations (15 for testing, 50+ for a full run.)
 config.maxIterBeforeFinalCycle = 30

cookbook/fgcmMakeLutHscFromModtran.py

@@ -1,11 +1,13 @@
 """
 HSC-specific overrides for FgcmMakeLut
 """
-
-# Short-code Filter names
-config.filterNames = ('g', 'r', 'i', 'z', 'y')
-# Each filter maps onto a "standard filter".
-config.stdFilterNames = ('g', 'r', 'i', 'z', 'y')
+# List of physical filter labels to build LUT
+config.physicalFilters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
+# For RC2, we must explicitly map HSC-R to HSC-R and HSC-I to HSC-I
+# in the override configuration, as the obs_subaru default assumes both
+# HSC-R + HSC-R2 and HSC-I + HSC-I2 data
+config.stdPhysicalFilterOverrideMap = {'HSC-R': 'HSC-R',
+                                       'HSC-I': 'HSC-I'}
 
 # Telescope elevation in meters
 config.parameters.elevation = 4139.0

cookbook/fgcmMakeLutHscFromTable.py

@@ -1,11 +1,13 @@
 """
 HSC-specific overrides for FgcmMakeLut
 """
-
-# Short-code Filter names
-config.filterNames = ('g', 'r', 'i', 'z', 'y')
-# Each filter maps onto a "standard filter".
-config.stdFilterNames = ('g', 'r', 'i', 'z', 'y')
+# Physical filter labels to generate LUT
+config.physicalFilters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
+# For RC2, we must explicitly map HSC-R to HSC-R and HSC-I to HSC-I
+# in the override configuration, as the obs_subaru default assumes both
+# HSC-R + HSC-R2 and HSC-I + HSC-I2 data
+config.stdPhysicalFilterOverrideMap = {'HSC-R': 'HSC-R',
+                                       'HSC-I': 'HSC-I'}
 # Pre-generated atmosphere table distributed with FGCM
 config.atmosphereTableName = 'fgcm_atm_subaru3'
 

python/lsst/fgcmcal/fgcmBuildStarsBase.py

@@ -100,7 +100,9 @@ class FgcmBuildStarsConfigBase(pexConfig.Config):
                     "DM-28088.  It will be removed after v22.  Use "
                     "physicalFilterMap instead.")
     )
-    # The following will not be necessary after Gen2 retirement.
+    # The following config will not be necessary after Gen2 retirement.
+    # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
+    # which is easiest to access in the config file.
     physicalFilterMap = pexConfig.DictField(
         doc="Mapping from 'physicalFilter' to band.",
         keytype=str,
@@ -564,6 +566,8 @@ def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
                 psf = exp.getPsf()
             else:
                 visitInfo = dataRef.get(component='visitInfo')
+                # TODO: When DM-28583 is fixed we can get the filterLabel
+                # via dataRef.get(component='filterLabel')
                 physicalFilter = dataRef.dataId['physical_filter']
                 psf = dataRef.get(component='psf')
 
@@ -733,7 +737,6 @@ def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
             referenceFilterNames = []
 
         # make the fgcm starConfig dict
-        print(self.config.physicalFilterMap)
         starConfig = {'logger': self.log,
                       'filterToBand': self.config.physicalFilterMap,
                       'requiredBands': self.config.requiredBands,

python/lsst/fgcmcal/fgcmFitCycle.py

@@ -246,7 +246,9 @@ class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
                     "DM-28088.  It will be removed after v22.  Use "
                     "physicalFilterMap instead.")
     )
-    # The following will not be necessary after Gen2 retirement.
+    # The following config will not be necessary after Gen2 retirement.
+    # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
+    # is easiest to access in the config file.
     physicalFilterMap = pexConfig.DictField(
         doc="Mapping from 'physicalFilter' to band.",
         keytype=str,

python/lsst/fgcmcal/fgcmLoadReferenceCatalog.py

@@ -57,6 +57,15 @@ class FgcmLoadReferenceCatalogConfig(pexConfig.Config):
         keytype=str,
         itemtype=str,
         default={},
+        deprecated=("This field is no longer used, and has been deprecated by "
+                    "DM-28088.  It will be removed after v22.  Use "
+                    "filterMap instead.")
+    )
+    filterMap = pexConfig.DictField(
+        doc="Mapping from physicalFilter label to reference filter name.",
+        keytype=str,
+        itemtype=str,
+        default={},
     )
     applyColorTerms = pexConfig.Field(
         doc=("Apply photometric color terms to reference stars?"
@@ -75,9 +84,9 @@ class FgcmLoadReferenceCatalogConfig(pexConfig.Config):
 
     def validate(self):
         super().validate()
-        if not self.refFilterMap:
-            msg = 'Must set refFilterMap'
-            raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.refFilterMap, self, msg)
+        if not self.filterMap:
+            msg = 'Must set filterMap'
+            raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.filterMap, self, msg)
         if self.applyColorTerms and len(self.colorterms.data) == 0:
             msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
             raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.colorterms, self, msg)
@@ -308,7 +317,7 @@ def _determine_flux_fields(self, center, filterList):
         # via the refObjLoader task which requires a valid filterName
         foundReferenceFilter = False
         for filterName in filterList:
-            refFilterName = self.config.refFilterMap.get(filterName)
+            refFilterName = self.config.filterMap.get(filterName)
             if refFilterName is None:
                 continue
 
@@ -333,7 +342,7 @@ def _determine_flux_fields(self, center, filterList):
         for filterName in filterList:
             fluxField = None
 
-            refFilterName = self.config.refFilterMap.get(filterName)
+            refFilterName = self.config.filterMap.get(filterName)
 
             if refFilterName is not None:
                 try:

python/lsst/fgcmcal/fgcmMakeLut.py

@@ -252,14 +252,18 @@ class FgcmMakeLutConfig(pipeBase.PipelineTaskConfig,
                     "DM-28088.  It will be removed after v22.  Use "
                     "stdPhysicalFilterMap instead.")
     )
-    stdPhysicalFilterMap = pexConfig.DictField(
-        doc=("Mapping from physical filter names to 'standard' physical filter "
-             "names.  Each physical filter key in this dict will be calibrated "
-             "to the matched standard physical filter.  In generic usage, one "
-             "would have (e.g.) g->g, r->r, etc.  In the case of HSC, one would "
-             "have HSC-G->HSC-G, HSC-R->HSC-R2, HSC-R2->HSC-R2 which allows "
-             "replacement (or time-variable) filters to be properly "
-             "cross-calibrated to a specific bandpass."),
+    physicalFilters = pexConfig.ListField(
+        doc="List of physicalFilter labels to generate look-up table.",
+        dtype=str,
+        default=[],
+    )
+    stdPhysicalFilterOverrideMap = pexConfig.DictField(
+        doc=("Override mapping from physical filter labels to 'standard' physical "
+             "filter labels. The 'standard' physical filter defines the transmission "
+             "curve that the FGCM standard bandpass will be based on. "
+             "Any filter not listed here will be mapped to "
+             "itself (e.g. g->g or HSC-G->HSC-G).  Use this override for cross-"
+             "filter calibration such as HSC-R->HSC-R2 and HSC-I->HSC-I2."),
         keytype=str,
         itemtype=str,
         default={},
@@ -286,7 +290,8 @@ def validate(self):
         directly from the specified atmosphereTableName.
         """
         # check that filterNames and stdFilterNames are okay
-        self._fields['stdPhysicalFilterMap'].validate(self)
+        self._fields['physicalFilters'].validate(self)
+        self._fields['stdPhysicalFilterOverrideMap'].validate(self)
 
         if self.atmosphereTableName is None:
             # Validate the parameters
@@ -405,7 +410,7 @@ def runDataRef(self, butler):
                                                                  dataId={'ccd': detector.getId()})
 
         filterDataRefDict = {}
-        for physicalFilter in self.config.stdPhysicalFilterMap:
+        for physicalFilter in self.config.physicalFilters:
             # The physical filters map directly to dataId filter names
             # for gen2 HSC.  This is the only camera that will be supported
             # by Gen2 fgcmcal, so we do not need to worry about other cases.
@@ -433,7 +438,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
                              sensorRef in sensorRefs}
 
         filterRefs = butlerQC.get(inputRefs.transmission_filter)
-        filterDataRefDict = {filterRef.dataId.byName()['physical_filter']: filterRef for
+        filterDataRefDict = {filterRef.dataId['physical_filter']: filterRef for
                              filterRef in filterRefs}
 
         lutCat = self._fgcmMakeLut(camera,
@@ -461,7 +466,7 @@ def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict,
         filterDataRefDict : `dict` of [`str`, `lsst.daf.persistence.ButlerDataRef` or
                             `lsst.daf.butler.DeferredDatasetHandle`]
             Dictionary of references to filter transmission curves.  Key will
-            be physical filter name.
+            be physical filter label.
 
         Returns
         -------
@@ -498,7 +503,7 @@ def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict,
                        throughputLambda[1] - throughputLambda[0]))
 
         throughputDict = {}
-        for i, physicalFilter in enumerate(self.config.stdPhysicalFilterMap):
+        for i, physicalFilter in enumerate(self.config.physicalFilters):
             tDict = {}
             tDict['LAMBDA'] = throughputLambda
             for ccdIndex, detector in enumerate(camera):
@@ -516,13 +521,8 @@ def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict,
 
         # build the index values
         comma = ','
-        physicalFilters = []
-        stdPhysicalFilters = []
-        for physicalFilter in self.config.stdPhysicalFilterMap:
-            physicalFilters.append(physicalFilter)
-            stdPhysicalFilters.append(self.config.stdPhysicalFilterMap[physicalFilter])
-        physicalFilterString = comma.join(physicalFilters)
-        stdPhysicalFilterString = comma.join(stdPhysicalFilters)
+        physicalFilterString = comma.join(self.config.physicalFilters)
+        stdPhysicalFilterString = comma.join(self._getStdPhysicalFilterList())
 
         atmosphereTableName = 'NoTableWasUsed'
         if self.config.atmosphereTableName is not None:
@@ -535,6 +535,18 @@ def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict,
                                   stdPhysicalFilterString, atmosphereTableName)
         return lutCat
 
+    def _getStdPhysicalFilterList(self):
+        """Get the standard physical filter lists from config.physicalFilters
+        and config.stdPhysicalFilterOverrideMap
+
+        Returns
+        -------
+        stdPhysicalFilters : `list`
+        """
+        override = self.config.stdPhysicalFilterOverrideMap
+        return [override.get(physicalFilter, physicalFilter) for
+                physicalFilter in self.config.physicalFilters]
+
     def _createLutConfig(self, nCcd):
         """
         Create the fgcmLut config dictionary
@@ -548,13 +560,8 @@ def _createLutConfig(self, nCcd):
         # create the common stub of the lutConfig
         lutConfig = {}
         lutConfig['logger'] = self.log
-        physicalFilters = []
-        stdPhysicalFilters = []
-        for physicalFilter in self.config.stdPhysicalFilterMap:
-            physicalFilters.append(physicalFilter)
-            stdPhysicalFilters.append(self.config.stdPhysicalFilterMap[physicalFilter])
-        lutConfig['filterNames'] = physicalFilters
-        lutConfig['stdFilterNames'] = stdPhysicalFilters
+        lutConfig['filterNames'] = self.config.physicalFilters
+        lutConfig['stdFilterNames'] = self._getStdPhysicalFilterList()
         lutConfig['nCCD'] = nCcd
 
         # atmosphereTable already validated if available
@@ -604,7 +611,7 @@ def _loadThroughputs(self, camera, opticsDataRef, sensorDataRefDict, filterDataR
         filterDataRefDict : `dict` of [`str`, `lsst.daf.persistence.ButlerDataRef` or
                             `lsst.daf.butler.DeferredDatasetHandle`]
             Dictionary of references to filter transmission curves.  Key will
-            be physical filter name.
+            be physical filter label.
 
         Raises
         ------
@@ -618,7 +625,7 @@ def _loadThroughputs(self, camera, opticsDataRef, sensorDataRefDict, filterDataR
             self._sensorsTransmission[detector.getId()] = sensorDataRefDict[detector.getId()].get()
 
         self._filtersTransmission = {}
-        for physicalFilter in self.config.stdPhysicalFilterMap:
+        for physicalFilter in self.config.physicalFilters:
             self._filtersTransmission[physicalFilter] = filterDataRefDict[physicalFilter].get()
 
     def _getThroughputDetector(self, detector, physicalFilter, throughputLambda):
@@ -631,7 +638,7 @@ def _getThroughputDetector(self, detector, physicalFilter, throughputLambda):
         detector: `lsst.afw.cameraGeom._detector.Detector`
            Detector on camera
         physicalFilter: `str`
-           Physical filter name
+           Physical filter label
         throughputLambda: `np.array(dtype=np.float64)`
            Wavelength steps (Angstrom)
 

python/lsst/fgcmcal/utilities.py

@@ -367,7 +367,7 @@ def translateVisitCatalog(visitCat):
                                                  ('TELDEC', 'f8'),
                                                  ('TELROT', 'f8'),
                                                  ('PMB', 'f8'),
-                                                 ('FILTERNAME', 'a30')])
+                                                 ('FILTERNAME', 'a50')])
     fgcmExpInfo['VISIT'][:] = visitCat['visit']
     fgcmExpInfo['MJD'][:] = visitCat['mjd']
     fgcmExpInfo['EXPTIME'][:] = visitCat['exptime']

tests/config/fgcmMakeLutHsc.py

@@ -1,4 +1,5 @@
-config.stdPhysicalFilterMap = {'HSC-G': 'HSC-G',
-                               'HSC-R': 'HSC-R',
-                               'HSC-I': 'HSC-I'}
+config.physicalFilters = ['HSC-G', 'HSC-R', 'HSC-I']
+# We need the physical filter overrides here to override obs_subaru configs
+config.stdPhysicalFilterOverrideMap = {'HSC-R': 'HSC-R',
+                                       'HSC-I': 'HSC-I'}
 config.atmosphereTableName = 'fgcm_atm_subaru2_test'

tests/test_loadref_hsc.py

@@ -64,7 +64,7 @@ def test_fgcmLoadReference(self):
         config = fgcmcal.FgcmLoadReferenceCatalogConfig()
         config.applyColorTerms = True
         config.refObjLoader.ref_dataset_name = 'ps1_pv3_3pi_20170110'
-        config.refFilterMap = {'HSC-R': 'r', 'HSC-I': 'i'}
+        config.filterMap = {'HSC-R': 'r', 'HSC-I': 'i'}
         config.colorterms.data = {}
         config.colorterms.data['ps1*'] = lsst.pipe.tasks.colorterms.ColortermDict()
         config.colorterms.data['ps1*'].data = {}
@@ -124,7 +124,7 @@ def test_fgcmLoadReferenceOtherFilters(self):
         config = fgcmcal.FgcmLoadReferenceCatalogConfig()
         config.applyColorTerms = True
         config.refObjLoader.ref_dataset_name = 'ps1_pv3_3pi_20170110'
-        config.refFilterMap = {'HSC-R2': 'r', 'HSC-I2': 'i'}
+        config.filterMap = {'HSC-R2': 'r', 'HSC-I2': 'i'}
         config.colorterms.data = {}
         config.colorterms.data['ps1*'] = lsst.pipe.tasks.colorterms.ColortermDict()
         config.colorterms.data['ps1*'].data = {}