Merge pull request #59 from lsst/tickets/DM-29348

DM-29348: Add ability to run global fgcmcal in a single pipeline.

python/lsst/fgcmcal/fgcmCalibrateTractBase.py

@@ -81,6 +81,7 @@ def setDefaults(self):
         pexConfig.Config.setDefaults(self)
 
         self.fgcmFitCycle.quietMode = True
+        self.fgcmFitCycle.doPlots = False
         self.fgcmOutputProducts.doReferenceCalibration = False
         self.fgcmOutputProducts.doRefcatOutput = False
         self.fgcmOutputProducts.cycleNumber = 0
@@ -379,9 +380,6 @@ def run(self, dataRefDict, tract,
                                     True, False, lutIndexVals[0]['FILTERNAMES'],
                                     tract=tract)
 
-        # Turn off plotting in tract mode
-        configDict['doPlots'] = False
-
         # Use the first orientation.
         # TODO: DM-21215 will generalize to arbitrary camera orientations
         ccdOffsets = computeCcdOffsets(dataRefDict['camera'], fgcmExpInfo['TELROT'][0])

python/lsst/fgcmcal/fgcmOutputProducts.py

@@ -85,31 +85,31 @@ class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
         deferLoad=True,
     )
 
-    fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
+    fgcmVisitCatalog = connectionTypes.Input(
         doc="Catalog of visit information for fgcm",
         name="fgcmVisitCatalog",
         storageClass="Catalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmStandardStars = connectionTypes.PrerequisiteInput(
+    fgcmStandardStars = connectionTypes.Input(
         doc="Catalog of standard star data from fgcm fit",
         name="fgcmStandardStars{cycleNumber}",
         storageClass="SimpleCatalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmZeropoints = connectionTypes.PrerequisiteInput(
+    fgcmZeropoints = connectionTypes.Input(
         doc="Catalog of zeropoints from fgcm fit",
         name="fgcmZeropoints{cycleNumber}",
         storageClass="Catalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
+    fgcmAtmosphereParameters = connectionTypes.Input(
         doc="Catalog of atmosphere parameters from fgcm fit",
         name="fgcmAtmosphereParameters{cycleNumber}",
         storageClass="Catalog",
@@ -126,12 +126,6 @@ class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
         multiple=True,
     )
 
-    fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
-        doc="Config used to build FGCM input stars",
-        name="fgcmBuildStarsTable_config",
-        storageClass="Config",
-    )
-
     fgcmPhotoCalib = connectionTypes.Output(
         doc=("Per-visit photometric calibrations derived from fgcm calibration. "
              "These catalogs use detector id for the id and are sorted for "
@@ -173,9 +167,9 @@ def __init__(self, *, config=None):
         if not config.doReferenceCalibration:
             self.prerequisiteInputs.remove("refCat")
         if not config.doAtmosphereOutput:
-            self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
+            self.inputs.remove("fgcmAtmosphereParameters")
         if not config.doZeropointOutput:
-            self.prerequisiteInputs.remove("fgcmZeropoints")
+            self.inputs.remove("fgcmZeropoints")
         if not config.doReferenceCalibration:
             self.outputs.remove("fgcmOffsets")
 
@@ -189,7 +183,12 @@ class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
         dtype=int,
         default=None,
     )
-
+    physicalFilterMap = pexConfig.DictField(
+        doc="Mapping from 'physicalFilter' to band.",
+        keytype=str,
+        itemtype=str,
+        default={},
+    )
     # The following fields refer to calibrating from a reference
     # catalog, but in the future this might need to be expanded
     doReferenceCalibration = pexConfig.Field(
@@ -408,18 +407,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
         else:
             self.refObjLoader = None
 
-        dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
-
-        fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
-        physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
-
-        if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
-            raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
-                               "in fgcmBuildStarsTask.")
-        if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
-            self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
-
-        struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
+        struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=True)
 
         # Output the photoCalib exposure catalogs
         if struct.photoCalibCatalogs is not None:

python/lsst/fgcmcal/utilities.py

@@ -25,6 +25,7 @@
 """
 
 import numpy as np
+import os
 import re
 
 from lsst.daf.base import PropertyList
@@ -37,7 +38,6 @@
 from lsst.obs.base import createInitialSkyWcs
 from lsst.obs.base import Instrument
 
-
 import fgcm
 
 
@@ -203,9 +203,11 @@ def makeConfigDict(config, log, camera, maxIter,
                   'quietMode': config.quietMode,
                   'randomSeed': config.randomSeed,
                   'outputStars': False,
+                  'outputPath': os.path.abspath('.'),
                   'clobber': True,
                   'useSedLUT': False,
                   'resetParameters': resetFitParameters,
+                  'doPlots': config.doPlots,
                   'outputFgcmcalZpts': True,  # when outputting zpts, use fgcmcal format
                   'outputZeropoints': outputZeropoints}
 

tests/config/fgcmBuildStarsHsc.py

@@ -1,4 +1,3 @@
-config.physicalFilterMap = {'HSC-G': 'g', 'HSC-R': 'r', 'HSC-I': 'i'}
 config.requiredBands = ['r', 'i']
 config.primaryBands = ['i']
 config.minPerBand = 2

tests/config/fgcmFitCycleHsc.py

@@ -1,12 +1,6 @@
 import lsst.fgcmcal as fgcmcal
 
 config.outfileBase = 'TestFgcm'
-# The unused z-band is here to test the case that extra filters
-# are in the map that are not used in the calibrations.
-config.physicalFilterMap = {'HSC-G': 'g',
-                            'HSC-R': 'r',
-                            'HSC-I': 'i',
-                            'HSC-Z': 'z'}
 config.bands = ['g', 'r', 'i']
 config.fitBands = ['g', 'r', 'i']
 config.requiredBands = ['r', 'i']