Add new ability for FgcmFitCycleTask to run multiple cycles.

FGCM can now be configured to run the full pipeline (build
stars, fit multiple cycles, and output) in one run.  The gen3
testing has also been refactored to use the same repository for
all the tests.

python/lsst/fgcmcal/fgcmCalibrateTractBase.py

@@ -81,6 +81,7 @@ def setDefaults(self):
         pexConfig.Config.setDefaults(self)
 
         self.fgcmFitCycle.quietMode = True
+        self.fgcmFitCycle.doPlots = False
         self.fgcmOutputProducts.doReferenceCalibration = False
         self.fgcmOutputProducts.doRefcatOutput = False
         self.fgcmOutputProducts.cycleNumber = 0
@@ -379,9 +380,6 @@ def run(self, dataRefDict, tract,
                                     True, False, lutIndexVals[0]['FILTERNAMES'],
                                     tract=tract)
 
-        # Turn off plotting in tract mode
-        configDict['doPlots'] = False
-
         # Use the first orientation.
         # TODO: DM-21215 will generalize to arbitrary camera orientations
         ccdOffsets = computeCcdOffsets(dataRefDict['camera'], fgcmExpInfo['TELROT'][0])

python/lsst/fgcmcal/fgcmOutputProducts.py

@@ -85,31 +85,31 @@ class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
         deferLoad=True,
     )
 
-    fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
+    fgcmVisitCatalog = connectionTypes.Input(
         doc="Catalog of visit information for fgcm",
         name="fgcmVisitCatalog",
         storageClass="Catalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmStandardStars = connectionTypes.PrerequisiteInput(
+    fgcmStandardStars = connectionTypes.Input(
         doc="Catalog of standard star data from fgcm fit",
         name="fgcmStandardStars{cycleNumber}",
         storageClass="SimpleCatalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmZeropoints = connectionTypes.PrerequisiteInput(
+    fgcmZeropoints = connectionTypes.Input(
         doc="Catalog of zeropoints from fgcm fit",
         name="fgcmZeropoints{cycleNumber}",
         storageClass="Catalog",
         dimensions=("instrument",),
         deferLoad=True,
     )
 
-    fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput(
+    fgcmAtmosphereParameters = connectionTypes.Input(
         doc="Catalog of atmosphere parameters from fgcm fit",
         name="fgcmAtmosphereParameters{cycleNumber}",
         storageClass="Catalog",
@@ -126,12 +126,6 @@ class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections,
         multiple=True,
     )
 
-    fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput(
-        doc="Config used to build FGCM input stars",
-        name="fgcmBuildStarsTable_config",
-        storageClass="Config",
-    )
-
     fgcmPhotoCalib = connectionTypes.Output(
         doc=("Per-visit photometric calibrations derived from fgcm calibration. "
              "These catalogs use detector id for the id and are sorted for "
@@ -173,9 +167,9 @@ def __init__(self, *, config=None):
         if not config.doReferenceCalibration:
             self.prerequisiteInputs.remove("refCat")
         if not config.doAtmosphereOutput:
-            self.prerequisiteInputs.remove("fgcmAtmosphereParameters")
+            self.inputs.remove("fgcmAtmosphereParameters")
         if not config.doZeropointOutput:
-            self.prerequisiteInputs.remove("fgcmZeropoints")
+            self.inputs.remove("fgcmZeropoints")
         if not config.doReferenceCalibration:
             self.outputs.remove("fgcmOffsets")
 
@@ -189,7 +183,12 @@ class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig,
         dtype=int,
         default=None,
     )
-
+    physicalFilterMap = pexConfig.DictField(
+        doc="Mapping from 'physicalFilter' to band.",
+        keytype=str,
+        itemtype=str,
+        default={},
+    )
     # The following fields refer to calibrating from a reference
     # catalog, but in the future this might need to be expanded
     doReferenceCalibration = pexConfig.Field(
@@ -408,18 +407,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
         else:
             self.refObjLoader = None
 
-        dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
-
-        fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig)
-        physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
-
-        if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
-            raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied "
-                               "in fgcmBuildStarsTask.")
-        if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
-            self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.")
-
-        struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True)
+        struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=True)
 
         # Output the photoCalib exposure catalogs
         if struct.photoCalibCatalogs is not None:

python/lsst/fgcmcal/utilities.py

@@ -25,6 +25,7 @@
 """
 
 import numpy as np
+import os
 import re
 
 from lsst.daf.base import PropertyList
@@ -37,7 +38,6 @@
 from lsst.obs.base import createInitialSkyWcs
 from lsst.obs.base import Instrument
 
-
 import fgcm
 
 
@@ -203,9 +203,11 @@ def makeConfigDict(config, log, camera, maxIter,
                   'quietMode': config.quietMode,
                   'randomSeed': config.randomSeed,
                   'outputStars': False,
+                  'outputPath': os.path.abspath('.'),
                   'clobber': True,
                   'useSedLUT': False,
                   'resetParameters': resetFitParameters,
+                  'doPlots': config.doPlots,
                   'outputFgcmcalZpts': True,  # when outputting zpts, use fgcmcal format
                   'outputZeropoints': outputZeropoints}
 

tests/config/fgcmBuildStarsHsc.py

@@ -1,4 +1,3 @@
-config.physicalFilterMap = {'HSC-G': 'g', 'HSC-R': 'r', 'HSC-I': 'i'}
 config.requiredBands = ['r', 'i']
 config.primaryBands = ['i']
 config.minPerBand = 2

tests/config/fgcmFitCycleHsc.py

@@ -1,12 +1,6 @@
 import lsst.fgcmcal as fgcmcal
 
 config.outfileBase = 'TestFgcm'
-# The unused z-band is here to test the case that extra filters
-# are in the map that are not used in the calibrations.
-config.physicalFilterMap = {'HSC-G': 'g',
-                            'HSC-R': 'r',
-                            'HSC-I': 'i',
-                            'HSC-Z': 'z'}
 config.bands = ['g', 'r', 'i']
 config.fitBands = ['g', 'r', 'i']
 config.requiredBands = ['r', 'i']