Merge pull request #48 from lsst/tickets/DM-28088

DM-28088: Replace uses of filter with physicalFilter as appropriate.

cookbook/README.md

@@ -64,16 +64,19 @@ filters aren't replaced that often.)
 ### Constructing a LUT from a precomputed atmosphere table
 
 A [sample config](fgcmMakeLutHscFromTable.py) for HSC in in the `cookbook/`
-directory.  There are only 3 fields to know about.  `filterNames` is a list of
-the "canonical" filter names.  Multiple "filters" can reference a single "band"
-(for example as the `r` and `i` filters have been replaced on HSC).  All filters
-that map to a single band will be calibrated together and put on the same
-system.  The `stdFilterNames` field is a list which matches each filter to a
-"standard filter".  For example, you can specify that all observations taken
-with the `i` filter should be transformed to the `i2` filter.  Note that at the
-current time the RC dataset on `lsst-dev` does not contain any `i2` data.  Finally,
-`atmosphereTableName` specifies the name of the table distributed with the
-third-party FGCM code; see that code for details.
+directory.  There are only 3 fields to know about.  `physicalFilters` is a list
+of the physical filter labels to make the LUT.  Multiple physical filters can
+reference a single band (for example, HSC-R and HSC-R2 both refer to the same
+r-band).  All filters that map to a single band will be calibrated together and
+put on the same system.  The `stdPhysicalFilterOverrideMap` dictionary field
+is the override mapping from physical filter labels to "standard" physical
+filter labels.  The "standard" physical filter defines the transmission curve
+that the FGCM standard bandpass will be based on.  Any filter not listed here
+will be mapped to itself (e.g. HSC-G->HSC-G).  Use this override for cross-
+filter calibration such as HSC-R->HSC-R2 and HSC-I->HSC-I2.  Note that at the
+current time the RC2 dataset on `lsst-devl` does not contain any HSC-R2 or
+HSC-I2 data.  Finally, `atmosphereTableName` specifies the name of the table
+distributed with the third-party FGCM code; see that code for details.
 
 ### Constructing a LUT using MODTRAN4
 

cookbook/fgcmBuildStarsHsc.py

@@ -18,9 +18,6 @@
 config.densityCutNside = 128
 # If there are more than densityCutMaxPerPixel stars per pixel, sample them
 config.densityCutMaxPerPixel = 1500
-# Dictionary that maps "filters" (instrumental configurations) to "bands"
-# (abstract names).  All filters must be listed in the LUT.
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
 # Which bands are required to be observed to be considered a calibration star
 config.requiredBands = ['g', 'r', 'i', 'z']
 # The reference CCD is a good CCD used to select visit to speed up the scanning

cookbook/fgcmBuildStarsTableHsc.py

@@ -16,9 +16,6 @@
 config.densityCutNside = 128
 # If there are more than densityCutMaxPerPixel stars per pixel, sample them
 config.densityCutMaxPerPixel = 1500
-# Dictionary that maps "filters" (instrumental configurations) to "bands"
-# (abstract names).  All filters must be listed in the LUT.
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
 # Which bands are required to be observed to be considered a calibration star
 config.requiredBands = ['g', 'r', 'i', 'z']
 # The reference CCD is a good CCD used to select visit to speed up the scanning

cookbook/fgcmFitCycleHscCookbook_cycle00_config.py

@@ -12,7 +12,7 @@
 config.fitBands = ('g', 'r', 'i', 'z', 'y')
 # Bands that are required for a star to be considered a calibration star.
 config.requiredBands = ('g', 'r', 'i', 'z', 'y')
-config.filterMap = {'g': 'g', 'r': 'r', 'i': 'i', 'z': 'z', 'y': 'y'}
+
 
 # Maximum number of fit iterations (15 for testing, 50+ for a full run.)
 config.maxIterBeforeFinalCycle = 30

cookbook/fgcmMakeLutHscFromModtran.py

@@ -1,11 +1,13 @@
 """
 HSC-specific overrides for FgcmMakeLut
 """
-
-# Short-code Filter names
-config.filterNames = ('g', 'r', 'i', 'z', 'y')
-# Each filter maps onto a "standard filter".
-config.stdFilterNames = ('g', 'r', 'i', 'z', 'y')
+# List of physical filter labels to build LUT
+config.physicalFilters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
+# For RC2, we must explicitly map HSC-R to HSC-R and HSC-I to HSC-I
+# in the override configuration, as the obs_subaru default assumes both
+# HSC-R + HSC-R2 and HSC-I + HSC-I2 data
+config.stdPhysicalFilterOverrideMap = {'HSC-R': 'HSC-R',
+                                       'HSC-I': 'HSC-I'}
 
 # Telescope elevation in meters
 config.parameters.elevation = 4139.0

cookbook/fgcmMakeLutHscFromTable.py

@@ -1,11 +1,13 @@
 """
 HSC-specific overrides for FgcmMakeLut
 """
-
-# Short-code Filter names
-config.filterNames = ('g', 'r', 'i', 'z', 'y')
-# Each filter maps onto a "standard filter".
-config.stdFilterNames = ('g', 'r', 'i', 'z', 'y')
+# Physical filter labels to generate LUT
+config.physicalFilters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
+# For RC2, we must explicitly map HSC-R to HSC-R and HSC-I to HSC-I
+# in the override configuration, as the obs_subaru default assumes both
+# HSC-R + HSC-R2 and HSC-I + HSC-I2 data
+config.stdPhysicalFilterOverrideMap = {'HSC-R': 'HSC-R',
+                                       'HSC-I': 'HSC-I'}
 # Pre-generated atmosphere table distributed with FGCM
 config.atmosphereTableName = 'fgcm_atm_subaru3'
 

python/lsst/fgcmcal/fgcmBuildStarsBase.py

@@ -96,6 +96,18 @@ class FgcmBuildStarsConfigBase(pexConfig.Config):
         keytype=str,
         itemtype=str,
         default={},
+        deprecated=("This field is no longer used, and has been deprecated by "
+                    "DM-28088.  It will be removed after v22.  Use "
+                    "physicalFilterMap instead.")
+    )
+    # The following config will not be necessary after Gen2 retirement.
+    # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
+    # which is easiest to access in the config file.
+    physicalFilterMap = pexConfig.DictField(
+        doc="Mapping from 'physicalFilter' to band.",
+        keytype=str,
+        itemtype=str,
+        default={},
     )
     requiredBands = pexConfig.ListField(
         doc="Bands required for each star",
@@ -549,16 +561,19 @@ def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
             if isinstance(dataRef, dafPersist.ButlerDataRef):
                 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
                 visitInfo = exp.getInfo().getVisitInfo()
-                f = exp.getFilter()
+                label = dataRef.get(datasetType='calexp_filterLabel')
+                physicalFilter = label.physicalLabel
                 psf = exp.getPsf()
             else:
                 visitInfo = dataRef.get(component='visitInfo')
-                f = dataRef.get(component='filter')
+                # TODO: When DM-28583 is fixed we can get the filterLabel
+                # via dataRef.get(component='filterLabel')
+                physicalFilter = dataRef.dataId['physical_filter']
                 psf = dataRef.get(component='psf')
 
             rec = visitCat[i]
             rec['visit'] = visit
-            rec['filtername'] = f.getName()
+            rec['physicalFilter'] = physicalFilter
             # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
             # much easier to get the wcs's necessary to recompute the pointing
             # ra/dec at the center of the camera.
@@ -688,9 +703,9 @@ def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
         """
         # get filter names into a numpy array...
         # This is the type that is expected by the fgcm code
-        visitFilterNames = np.zeros(len(visitCat), dtype='a10')
+        visitFilterNames = np.zeros(len(visitCat), dtype='a30')
         for i in range(len(visitCat)):
-            visitFilterNames[i] = visitCat[i]['filtername']
+            visitFilterNames[i] = visitCat[i]['physicalFilter']
 
         # match to put filterNames with observations
         visitIndex = np.searchsorted(visitCat['visit'],
@@ -703,10 +718,10 @@ def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
             lutCat = lutDataRef.get()
 
             stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
-                             zip(lutCat[0]['filterNames'].split(','),
-                                 lutCat[0]['stdFilterNames'].split(','))}
+                             zip(lutCat[0]['physicalFilters'].split(','),
+                                 lutCat[0]['stdPhysicalFilters'].split(','))}
             stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
-                             zip(lutCat[0]['stdFilterNames'].split(','),
+                             zip(lutCat[0]['stdPhysicalFilters'].split(','),
                                  lutCat[0]['lambdaStdFilter'])}
 
             del lutCat
@@ -722,9 +737,8 @@ def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
             referenceFilterNames = []
 
         # make the fgcm starConfig dict
-
         starConfig = {'logger': self.log,
-                      'filterToBand': self.config.filterMap,
+                      'filterToBand': self.config.physicalFilterMap,
                       'requiredBands': self.config.requiredBands,
                       'minPerBand': self.config.minPerBand,
                       'matchRadius': self.config.matchRadius,
@@ -824,8 +838,7 @@ def _makeFgcmVisitSchema(self, nCcd):
 
         schema = afwTable.Schema()
         schema.addField('visit', type=np.int32, doc="Visit number")
-        # Note that the FGCM code currently handles filternames up to 2 characters long
-        schema.addField('filtername', type=str, size=10, doc="Filter name")
+        schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
         schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
         schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
         schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
@@ -923,7 +936,7 @@ def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
         """
 
         # Find the unique list of filter names in visitCat
-        filterNames = np.unique(visitCat.asAstropy()['filtername'])
+        filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
 
         # Find the unique list of "standard" filters
         stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}

python/lsst/fgcmcal/fgcmCalibrateTractBase.py

@@ -240,20 +240,11 @@ def runDataRef(self, butler, dataRefs):
         if struct.photoCalibs is not None:
             self.log.info("Outputting photoCalib files.")
 
-            filterMapping = {}
-            for visit, detector, filtername, photoCalib in struct.photoCalibs:
-                if filtername not in filterMapping:
-                    # We need to find the mapping from encoded filter to dataid filter,
-                    # and this trick allows us to do that.
-                    dataId = {visitDataRefName: visit,
-                              ccdDataRefName: detector}
-                    dataRef = butler.dataRef('raw', dataId=dataId)
-                    filterMapping[filtername] = dataRef.dataId['filter']
-
+            for visit, detector, physicalFilter, photoCalib in struct.photoCalibs:
                 butler.put(photoCalib, 'fgcm_tract_photoCalib',
                            dataId={visitDataRefName: visit,
                                    ccdDataRefName: detector,
-                                   'filter': filterMapping[filtername],
+                                   'filter': physicalFilter,
                                    'tract': tract})
 
             self.log.info("Done outputting photoCalib files.")
@@ -380,7 +371,7 @@ def run(self, dataRefDict, tract,
         # Load the LUT
         lutCat = dataRefDict['fgcmLookUpTable'].get()
         fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
-                                                         dict(self.config.fgcmFitCycle.filterMap))
+                                                         dict(self.config.fgcmFitCycle.physicalFilterMap))
         del lutCat
 
         # Translate the visit catalog into fgcm format
@@ -426,7 +417,7 @@ def run(self, dataRefDict, tract,
 
         refMag, refMagErr = extractReferenceMags(fgcmRefCat,
                                                  self.config.fgcmFitCycle.bands,
-                                                 self.config.fgcmFitCycle.filterMap)
+                                                 self.config.fgcmFitCycle.physicalFilterMap)
         refId = fgcmRefCat['fgcm_id'][:]
 
         fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)

python/lsst/fgcmcal/fgcmFitCycle.py

@@ -242,6 +242,18 @@ class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
         keytype=str,
         itemtype=str,
         default={},
+        deprecated=("This field is no longer used, and has been deprecated by "
+                    "DM-28088.  It will be removed after v22.  Use "
+                    "physicalFilterMap instead.")
+    )
+    # The following config will not be necessary after Gen2 retirement.
+    # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
+    # is easiest to access in the config file.
+    physicalFilterMap = pexConfig.DictField(
+        doc="Mapping from 'physicalFilter' to band.",
+        keytype=str,
+        itemtype=str,
+        default={},
     )
     doReferenceCalibration = pexConfig.Field(
         doc="Use reference catalog as additional constraint on calibration",
@@ -1075,7 +1087,8 @@ def _fgcmFitCycle(self, camera, dataRefDict):
                                     self.outputZeropoints)
 
         lutCat = dataRefDict['fgcmLookUpTable'].get()
-        fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
+        fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
+                                                         dict(self.config.physicalFilterMap))
         del lutCat
 
         # next we need the exposure/visit information
@@ -1132,7 +1145,7 @@ def _fgcmFitCycle(self, camera, dataRefDict):
 
             refMag, refMagErr = extractReferenceMags(refStars,
                                                      self.config.bands,
-                                                     self.config.filterMap)
+                                                     self.config.physicalFilterMap)
             refId = refStars['fgcm_id'][:]
         else:
             refStars = None

python/lsst/fgcmcal/fgcmLoadReferenceCatalog.py

@@ -57,6 +57,15 @@ class FgcmLoadReferenceCatalogConfig(pexConfig.Config):
         keytype=str,
         itemtype=str,
         default={},
+        deprecated=("This field is no longer used, and has been deprecated by "
+                    "DM-28088.  It will be removed after v22.  Use "
+                    "filterMap instead.")
+    )
+    filterMap = pexConfig.DictField(
+        doc="Mapping from physicalFilter label to reference filter name.",
+        keytype=str,
+        itemtype=str,
+        default={},
     )
     applyColorTerms = pexConfig.Field(
         doc=("Apply photometric color terms to reference stars?"
@@ -75,9 +84,9 @@ class FgcmLoadReferenceCatalogConfig(pexConfig.Config):
 
     def validate(self):
         super().validate()
-        if not self.refFilterMap:
-            msg = 'Must set refFilterMap'
-            raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.refFilterMap, self, msg)
+        if not self.filterMap:
+            msg = 'Must set filterMap'
+            raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.filterMap, self, msg)
         if self.applyColorTerms and len(self.colorterms.data) == 0:
             msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
             raise pexConfig.FieldValidationError(FgcmLoadReferenceCatalogConfig.colorterms, self, msg)
@@ -308,7 +317,7 @@ def _determine_flux_fields(self, center, filterList):
         # via the refObjLoader task which requires a valid filterName
         foundReferenceFilter = False
         for filterName in filterList:
-            refFilterName = self.config.refFilterMap.get(filterName)
+            refFilterName = self.config.filterMap.get(filterName)
             if refFilterName is None:
                 continue
 
@@ -333,7 +342,7 @@ def _determine_flux_fields(self, center, filterList):
         for filterName in filterList:
             fluxField = None
 
-            refFilterName = self.config.refFilterMap.get(filterName)
+            refFilterName = self.config.filterMap.get(filterName)
 
             if refFilterName is not None:
                 try: