DM-28755: Update fgcmcal to make use of visitSummary tables

python/lsst/fgcmcal/fgcmBuildStars.py

@@ -104,12 +104,7 @@ def _makeArgumentParser(cls):
 
         return parser
 
-    def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
-        if butler is None:
-            raise RuntimeError("Gen2 _findAndGroupDataRefs must be called with a butler.")
-        if calexpDataRefDict is not None:
-            self.log.warn("Ignoring calexpDataRefDict in gen2 _findAndGroupDataRefs")
-
+    def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
         self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
 
         ccdIds = []

python/lsst/fgcmcal/fgcmBuildStarsBase.py

@@ -332,7 +332,7 @@ def runDataRef(self, butler, dataRefs):
                                    (self.config.instFluxField)) from e
 
         camera = butler.get('camera')
-        groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs, butler=butler)
+        groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs)
 
         # Make the visit catalog if necessary
         # First check if the visit catalog is in the _current_ path
@@ -393,35 +393,26 @@ def runDataRef(self, butler, dataRefs):
             butler.put(fgcmRefCat, 'fgcmReferenceStars')
 
     @abc.abstractmethod
-    def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
+    def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
         """
-        Find and group dataRefs (by visit).  For Gen2 usage, set butler, and for
-        Gen3, use calexpDataRefDict
+        Find and group dataRefs (by visit); Gen2 only.
 
         Parameters
         ----------
+        butler : `lsst.daf.persistence.Butler`
+            Gen2 butler.
         camera : `lsst.afw.cameraGeom.Camera`
             Camera from the butler.
-        dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or
-                   `lsst.daf.butler.DeferredDatasetHandle`
+        dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
             Data references for the input visits.
-        butler : `lsst.daf.persistence.Butler`, optional
-            Gen2 butler when used as CommandLineTask
-        calexpDataRefDict : `dict`, optional
-            Dictionary of Gen3 deferred data refs for calexps
 
         Returns
         -------
-        groupedDataRefs : `OrderedDict` [`int`, `list`]
+        groupedDataRefs : `dict` [`int`, `list`]
             Dictionary with sorted visit keys, and `list`s of
-            `lsst.daf.persistence.ButlerDataRef` or
-            `lsst.daf.butler.DeferredDatasetHandle`
-
-        Raises
-        ------
-        RuntimeError : Raised if neither or both of butler and dataRefDict are set.
+            `lsst.daf.persistence.ButlerDataRef`
         """
-        raise NotImplementedError("_findAndGroupDataRefs not implemented.")
+        raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
 
     @abc.abstractmethod
     def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
@@ -554,22 +545,38 @@ def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
                 if visitCatDataRef is not None:
                     visitCatDataRef.put(visitCat)
 
-            # Note that the reference ccd is first in the list (if available).
-
-            # The first dataRef in the group will be the reference ccd (if available)
             dataRef = groupedDataRefs[visit][0]
             if isinstance(dataRef, dafPersist.ButlerDataRef):
+                # Gen2: calexp dataRef
+                # The first dataRef in the group will be the reference ccd (if available)
                 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
                 visitInfo = exp.getInfo().getVisitInfo()
                 label = dataRef.get(datasetType='calexp_filterLabel')
                 physicalFilter = label.physicalLabel
                 psf = exp.getPsf()
+                psfSigma = psf.computeShape().getDeterminantRadius()
             else:
-                visitInfo = dataRef.get(component='visitInfo')
-                # TODO: When DM-28583 is fixed we can get the filterLabel
-                # via dataRef.get(component='filterLabel')
-                physicalFilter = dataRef.dataId['physical_filter']
-                psf = dataRef.get(component='psf')
+                # Gen3: use the visitSummary dataRef
+                summary = dataRef.get()
+
+                detectors = list(summary['detector_id'])
+
+                try:
+                    index = detectors.index(self.config.referenceCCD)
+                except ValueError:
+                    # Take first available ccd if reference isn't available
+                    index = 0
+
+                visitInfo = summary[index].getVisitInfo()
+                physicalFilter = summary[index]['physical_filter']
+                # Compute the median psf sigma if possible
+                goodSigma, = np.where(summary['psfSigma'] > 0)
+                if goodSigma.size > 2:
+                    psfSigma = np.median(summary['psfSigma'][goodSigma])
+                elif goodSigma > 0:
+                    psfSigma = np.mean(summary['psfSigma'][goodSigma])
+                else:
+                    psfSigma = 0.0
 
             rec = visitCat[i]
             rec['visit'] = visit
@@ -593,8 +600,7 @@ def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
             rec['scaling'][:] = 1.0
             # Median delta aperture, to be measured from stars
             rec['deltaAper'] = 0.0
-
-            rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
+            rec['psfSigma'] = psfSigma
 
             if self.config.doModelErrorsWithBackground:
                 foundBkg = False
@@ -604,7 +610,7 @@ def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
                         bgList = dataRef.get(datasetType='calexpBackground')
                         foundBkg = True
                 else:
-                    det = dataRef.dataId.byName()['detector']
+                    det = dataRef.dataId['detector']
                     try:
                         bkgRef = bkgDataRefDict[(visit, det)]
                         bgList = bkgRef.get()

python/lsst/fgcmcal/fgcmBuildStarsTable.py

@@ -94,11 +94,11 @@ class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
         multiple=True,
     )
 
-    calexp = connectionTypes.Input(
-        doc="Calibrated exposures used for psf and metadata",
-        name="calexp",
-        storageClass="ExposureF",
-        dimensions=("instrument", "visit", "detector"),
+    visitSummary = connectionTypes.Input(
+        doc="Per-visit summary statistics table",
+        name="visitSummary",
+        storageClass="ExposureCatalog",
+        dimensions=("instrument", "visit"),
         deferLoad=True,
         multiple=True,
     )
@@ -224,9 +224,13 @@ class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask):
     def runQuantum(self, butlerQC, inputRefs, outputRefs):
         inputRefDict = butlerQC.get(inputRefs)
 
-        dataRefs = inputRefDict['sourceTable_visit']
+        sourceTableRefs = inputRefDict['sourceTable_visit']
+
+        self.log.info("Running with %d sourceTable_visit dataRefs",
+                      len(sourceTableRefs))
 
-        self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefs)))
+        sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
+                                  sourceTableRef in sourceTableRefs}
 
         if self.config.doReferenceMatches:
             # Get the LUT dataRef
@@ -248,21 +252,20 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
         calibFluxApertureRadius = None
         if self.config.doSubtractLocalBackground:
             try:
-                calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
+                calibFluxApertureRadius = computeApertureRadiusFromDataRef(sourceTableRefs[0],
                                                                            self.config.instFluxField)
             except RuntimeError as e:
                 raise RuntimeError("Could not determine aperture radius from %s. "
                                    "Cannot use doSubtractLocalBackground." %
                                    (self.config.instFluxField)) from e
 
-        calexpRefs = inputRefDict['calexp']
-        calexpDataRefDict = {(calexpRef.dataId.byName()['visit'],
-                              calexpRef.dataId.byName()['detector']): calexpRef for
-                             calexpRef in calexpRefs}
+        visitSummaryRefs = inputRefDict['visitSummary']
+        visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
+                                   visitSummaryRef in visitSummaryRefs}
 
         camera = inputRefDict['camera']
-        groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs,
-                                                     calexpDataRefDict=calexpDataRefDict)
+        groupedDataRefs = self._groupDataRefs(sourceTableDataRefDict,
+                                              visitSummaryDataRefDict)
 
         if self.config.doModelErrorsWithBackground:
             bkgRefs = inputRefDict['background']
@@ -310,11 +313,34 @@ def _makeArgumentParser(cls):
 
         return parser
 
-    def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
-        if (butler is None and calexpDataRefDict is None) or \
-                (butler is not None and calexpDataRefDict is not None):
-            raise RuntimeError("Must either set butler (Gen2) or dataRefDict (Gen3)")
+    def _groupDataRefs(self, sourceTableDataRefDict, visitSummaryDataRefDict):
+        """Group sourceTable and visitSummary dataRefs (gen3 only).
+
+        Parameters
+        ----------
+        sourceTableDataRefDict : `dict` [`int`, `str`]
+            Dict of source tables, keyed by visit.
+        visitSummaryDataRefDict : `dict` [int, `str`]
+            Dict of visit summary catalogs, keyed by visit.
+
+        Returns
+        -------
+        groupedDataRefs : `dict` [`int`, `list`]
+            Dictionary with sorted visit keys, and `list`s with
+            `lsst.daf.butler.DeferredDataSetHandle`.  The first
+            item in the list will be the visitSummary ref, and
+            the second will be the source table ref.
+        """
+        groupedDataRefs = collections.defaultdict(list)
+        visits = sorted(sourceTableDataRefDict.keys())
+
+        for visit in visits:
+            groupedDataRefs[visit] = [visitSummaryDataRefDict[visit],
+                                      sourceTableDataRefDict[visit]]
+
+        return groupedDataRefs
 
+    def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
         self.log.info("Grouping dataRefs by %s", (self.config.visitDataRefName))
 
         ccdIds = []
@@ -337,19 +363,12 @@ def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict
             # Find an existing calexp (we need for psf and metadata)
             # and make the relevant dataRef
             for ccdId in ccdIds:
-                if butler is not None:
-                    # Gen2 Mode
-                    try:
-                        calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
-                                                                     self.config.ccdDataRefName: ccdId})
-                    except RuntimeError:
-                        # Not found
-                        continue
-                else:
-                    # Gen3 mode
-                    calexpRef = calexpDataRefDict.get((visit, ccdId))
-                    if calexpRef is None:
-                        continue
+                try:
+                    calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
+                                                                 self.config.ccdDataRefName: ccdId})
+                except RuntimeError:
+                    # Not found
+                    continue
 
                 # It was found.  Add and quit out, since we only
                 # need one calexp per visit.

python/lsst/fgcmcal/fgcmCalibrateTractBase.py

@@ -336,15 +336,12 @@ def run(self, dataRefDict, tract,
         # Note that we will need visitCat at the end of the procedure for the outputs
         if isinstance(butler, dafPersist.Butler):
             # Gen2
-            groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict['camera'],
-                                                                        dataRefDict['source_catalogs'],
-                                                                        butler=butler)
+            groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefsGen2(butler, dataRefDict['camera'],
+                                                                            dataRefDict['source_catalogs'])
         else:
             # Gen3
-            cdrd = dataRefDict['calexps']
-            groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict['camera'],
-                                                                        dataRefDict['source_catalogs'],
-                                                                        calexpDataRefDict=cdrd)
+            groupedDataRefs = self.fgcmBuildStars._groupDataRefs(dataRefDict['sourceTableDataRefDict'],
+                                                                 dataRefDict['visitSummaryDataRefDict'])
         visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict['camera'], groupedDataRefs)
         rad = calibFluxApertureRadius
         fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,

python/lsst/fgcmcal/fgcmCalibrateTractTable.py

@@ -84,11 +84,11 @@ class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections,
         multiple=True,
     )
 
-    calexp = connectionTypes.Input(
-        doc="Calibrated exposures used for psf and metadata",
-        name="calexp",
-        storageClass="ExposureF",
-        dimensions=("instrument", "visit", "detector"),
+    visitSummary = connectionTypes.Input(
+        doc="Per-visit summary statistics table",
+        name="visitSummary",
+        storageClass="ExposureCatalog",
+        dimensions=("instrument", "visit"),
         deferLoad=True,
         multiple=True,
     )
@@ -183,11 +183,16 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
         # Run the build stars tasks
         tract = butlerQC.quantum.dataId['tract']
 
-        calexpRefs = dataRefDict['calexp']
-        calexpRefDict = {(calexpRef.dataId.byName()['visit'],
-                          calexpRef.dataId.byName()['detector']):
-                         calexpRef for calexpRef in calexpRefs}
-        dataRefDict['calexps'] = calexpRefDict
+        sourceTableRefs = dataRefDict['source_catalogs']
+        sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for
+                                  sourceTableRef in sourceTableRefs}
+
+        visitSummaryRefs = dataRefDict['visitSummary']
+        visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for
+                                   visitSummaryRef in visitSummaryRefs}
+
+        dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict
+        dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict
 
         # And the outputs
         if self.config.fgcmOutputProducts.doZeropointOutput:
@@ -221,7 +226,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
             self.fgcmOutputProducts.refObjLoader = loader
 
         struct = self.run(dataRefDict, tract,
-                          buildStarsRefObjLoader=buildStarsRefObjLoader, butler=butlerQC)
+                          buildStarsRefObjLoader=buildStarsRefObjLoader)
 
         if struct.photoCalibCatalogs is not None:
             self.log.info("Outputting photoCalib catalogs.")

python/lsst/fgcmcal/fgcmOutputProducts.py

@@ -827,7 +827,7 @@ def _computeReferenceOffsets(self, stdCat, bands):
         results['hpix'] = ipring[rev[rev[gdpix]]]
 
         # We need a boolean index to deal with catalogs...
-        selected = np.zeros(len(stdCat), dtype=np.bool)
+        selected = np.zeros(len(stdCat), dtype=bool)
 
         refFluxFields = [None]*len(bands)
 
@@ -882,7 +882,7 @@ def _computeOffsetOneBand(self, sourceMapper, badStarKey,
            Name of band for reference catalog
         stdCat: `lsst.afw.table.SimpleCatalog`
            FGCM standard stars
-        selected: `numpy.array(dtype=np.bool)`
+        selected: `numpy.array(dtype=bool)`
            Boolean array of which stars are in the pixel
         refFluxFields: `list`
            List of names of flux fields for reference catalog
@@ -902,7 +902,7 @@ def _computeOffsetOneBand(self, sourceMapper, badStarKey,
             rec.set(badStarKey, True)
 
         exposure = afwImage.ExposureF()
-        exposure.setFilter(afwImage.Filter(band))
+        exposure.setFilterLabel(afwImage.FilterLabel(band=band))
 
         if refFluxFields[b] is None:
             # Need to find the flux field in the reference catalog
@@ -968,7 +968,7 @@ def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig):
         # Break up the pixels using a histogram
         h, rev = esutil.stat.histogram(indices, rev=True)
         gd, = np.where(h > 0)
-        selected = np.zeros(len(formattedCat), dtype=np.bool)
+        selected = np.zeros(len(formattedCat), dtype=bool)
         for i in gd:
             i1a = rev[rev[i]: rev[i + 1]]