Merge generic and HSC conf overrides in obs_subaru

SuprimeCam no longer supported. Remove config/hsc.

config/apertures.py

@@ -2,7 +2,7 @@
 # 'config' should be a SourceMeasurementConfig
 
 config.plugins.names |= ["base_CircularApertureFlux"]
-# Roughly (1.0, 1.4, 2.0, 2.8, 4.0, 5.7, 8.0, 11.3, 16.0, 22.6 arcsec) in diameter: 2**(0.5*i)
+# Roughly (1.0, 1.5, 2.0, 3.0, 4.0, 5.7, 8.4, 11.8, 16.8, 23.5 arcsec) in diameter: 2**(0.5*i)
 # (assuming plate scale of 0.168 arcsec pixels)
 config.plugins["base_CircularApertureFlux"].radii = [3.0, 4.5, 6.0, 9.0, 12.0, 17.0, 25.0, 35.0, 50.0, 70.0]
 

config/assembleCoadd.py

@@ -4,11 +4,16 @@
 config.load(os.path.join(os.path.dirname(__file__), "coaddBase.py"))
 
 config.doSigmaClip = False
-config.subregionSize = (10000, 200) # 200 rows (since patch width is typically < 10k pixels)
+# 200 rows (since patch width is typically < 10k pixels)
+config.subregionSize = (10000, 200)
 config.doMaskBrightObjects = True
 config.removeMaskPlanes.append("CROSSTALK")
 config.doNImage = True
 config.badMaskPlanes += ["SUSPECT"]
+config.doAttachTransmissionCurve = True
+# Saturation trails are usually oriented east-west, so along rows
+config.interpImage.transpose = True
+config.coaddPsf.warpingKernelName = 'lanczos5'
 
 from lsst.pipe.tasks.selectImages import PsfWcsSelectImagesTask
 config.select.retarget(PsfWcsSelectImagesTask)

config/hsc/bias.py → config/bias.py

@@ -1,6 +1,5 @@
 import os.path
 
-
 config.isr.load(os.path.join(os.path.dirname(__file__), "isr.py"))
 config.isr.doBrighterFatter = False
 config.isr.doStrayLight = False

config/calibrate.py

@@ -1,6 +1,7 @@
 import os.path
 
 from lsst.meas.algorithms import ColorLimit
+from lsst.meas.astrom import MatchOptimisticBConfig
 
 ObsConfigDir = os.path.dirname(__file__)
 
@@ -24,6 +25,21 @@
 # Better astrometry matching
 config.astrometry.matcher.numBrightStars = 150
 
+# Set to match defaults currently used in HSC production runs (e.g. S15B)
+config.astrometry.wcsFitter.numRejIter = 3
+config.astrometry.wcsFitter.order = 3
+
+for matchConfig in (config.astrometry,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxRotationDeg = 1.145916
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False
+
 # Set to match defaults curretnly used in HSC production runs (e.g. S15B)
 config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
 
@@ -35,6 +51,7 @@
 config.photoCal.match.referenceSelection.doMagLimit = True
 config.photoCal.match.referenceSelection.magLimit.fluxField = "i_flux"
 config.photoCal.match.referenceSelection.magLimit.maximum = 22.0
+config.photoCal.colorterms.load(os.path.join(ObsConfigDir, 'colorterms.py'))
 
 # Demand astrometry and photoCal succeed
 config.requireAstrometry = True
@@ -51,7 +68,9 @@
 
 # Deblender
 config.deblend.maxFootprintSize = 0
-config.deblend.maskLimits["NO_DATA"] = 0.25  # Ignore sources that are in the vignetted region
+# Ignore sources that are in the vignetted region
+config.deblend.maskLimits["NO_DATA"] = 0.25
 config.deblend.maxFootprintArea = 10000
 
 config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.168

config/charImage.py

@@ -1,6 +1,7 @@
 import os.path
 
 from lsst.meas.algorithms import ColorLimit
+from lsst.meas.astrom import MatchOptimisticBConfig
 
 ObsConfigDir = os.path.dirname(__file__)
 
@@ -15,11 +16,17 @@
 # PSF determination
 config.measurePsf.reserve.fraction = 0.2
 config.measurePsf.starSelector["objectSize"].sourceFluxField = "base_PsfFlux_instFlux"
+config.measurePsf.starSelector["objectSize"].widthMin = 0.9
+config.measurePsf.starSelector["objectSize"].fluxMin = 4000
 
 # Astrometry
 config.refObjLoader.load(os.path.join(ObsConfigDir, "filterMap.py"))
 config.refObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
 
+for refObjLoader in (config.refObjLoader,
+                     ):
+    refObjLoader.load(os.path.join(ObsConfigDir, "filterMap.py"))
+
 # Set to match defaults curretnly used in HSC production runs (e.g. S15B)
 config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
 
@@ -41,8 +48,34 @@
 config.deblend.maxFootprintArea = 10000
 
 config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.168
 
 # Convolved fluxes can fail for small target seeing if the observation seeing is larger
 if "ext_convolved_ConvolvedFlux" in config.measurement.plugins:
     names = config.measurement.plugins["ext_convolved_ConvolvedFlux"].getAllResultNames()
     config.measureApCorr.allowFailure += names
+
+# For aperture correction modeling, only use objects that were used in the
+# PSF model and have psf flux signal-to-noise > 200.
+config.measureApCorr.sourceSelector['science'].doFlags = True
+config.measureApCorr.sourceSelector['science'].doUnresolved = False
+config.measureApCorr.sourceSelector['science'].doSignalToNoise = True
+config.measureApCorr.sourceSelector['science'].flags.good = ["calib_psf_used"]
+config.measureApCorr.sourceSelector['science'].flags.bad = []
+config.measureApCorr.sourceSelector['science'].signalToNoise.minimum = 200.0
+config.measureApCorr.sourceSelector['science'].signalToNoise.maximum = None
+config.measureApCorr.sourceSelector['science'].signalToNoise.fluxField = "base_PsfFlux_instFlux"
+config.measureApCorr.sourceSelector['science'].signalToNoise.errField = "base_PsfFlux_instFluxErr"
+config.measureApCorr.sourceSelector.name = "science"
+
+config.ref_match.sourceSelector.name = 'matcher'
+for matchConfig in (config.ref_match,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxRotationDeg = 1.145916
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False

config/characterizeImage.py

@@ -1,7 +1,5 @@
 import os.path
 
-from lsst.meas.algorithms import ColorLimit
-
 ObsConfigDir = os.path.dirname(__file__)
 
 charImFile = os.path.join(ObsConfigDir, "charImage.py")

config/cmodel.py

@@ -1,6 +1,5 @@
 # Enable CModel mags (unsetup meas_modelfit to disable)
 # 'config' is a SourceMeasurementConfig.
-import os
 try:
     import lsst.meas.modelfit
     config.measurement.plugins.names |= ["modelfit_DoubleShapeletPsfApprox", "modelfit_CModel"]

config/hsc/coaddAnalysis.py → config/coaddAnalysis.py

@@ -1,6 +1,5 @@
 import os.path
 
-
 config.colorterms.load(os.path.join(os.path.dirname(__file__), "colorterms.py"))
 config.refObjLoader.load(os.path.join(os.path.dirname(__file__), "filterMap.py"))
 config.load(os.path.join(os.path.dirname(__file__), "srcSchemaMap.py"))

config/coaddBase.py

@@ -1,2 +1,5 @@
 # Configs shared between makeCoaddTempExp and assemble
 config.matchingKernelSize = 29
+config.doApplyExternalPhotoCalib = True
+config.externalPhotoCalibName = 'fgcm'
+config.doApplyExternalSkyWcs = True

config/coaddDriver.py

@@ -1,7 +1,6 @@
 # Load from sub-configurations
 import os.path
 
-
 from lsst.pipe.tasks.assembleCoadd import CompareWarpAssembleCoaddTask
 config.assembleCoadd.retarget(CompareWarpAssembleCoaddTask)
 

config/hsc/colorAnalysis.py → config/colorAnalysis.py

@@ -1,5 +1,4 @@
 import os.path
 
-
 config.load(os.path.join(os.path.dirname(__file__), "srcSchemaMap.py"))
 config.load(os.path.join(os.path.dirname(__file__), "extinctionCoeffs.py"))

config/hsc/colorterms.py → config/colorterms.py

@@ -1,5 +1,3 @@
-"""Set color terms for HSC"""
-
 from lsst.pipe.tasks.colorterms import Colorterm, ColortermDict
 
 config.data = {

config/hsc/compareCoaddAnalysis.py → config/compareCoaddAnalysis.py

@@ -1,6 +1,5 @@
 import os.path
 
-
 config.colorterms.load(os.path.join(os.path.dirname(__file__), "colorterms.py"))
 config.refObjLoader.load(os.path.join(os.path.dirname(__file__), "filterMap.py"))
 config.load(os.path.join(os.path.dirname(__file__), "srcSchemaMap.py"))

config/hsc/compareVisitAnalysis.py → config/compareVisitAnalysis.py

@@ -1,6 +1,4 @@
 import os.path
 
-
-config.refObjLoader.load(os.path.join(os.path.dirname(__file__), "..", "filterMap.py"))
 config.refObjLoader.load(os.path.join(os.path.dirname(__file__), "filterMap.py"))
 config.load(os.path.join(os.path.dirname(__file__), "srcSchemaMap.py"))

config/compareWarpAssembleCoadd.py

@@ -5,3 +5,6 @@
 
 # 200 rows (since patch width is typically < 10k pixels
 config.assembleStaticSkyModel.subregionSize = (10000, 200)
+config.assembleStaticSkyModel.doApplyExternalPhotoCalib = True
+config.assembleStaticSkyModel.externalPhotoCalibName = 'fgcm'
+config.assembleStaticSkyModel.doApplyExternalSkyWcs = True

config/hsc/convertRepo.py → config/convertRepo.py

@@ -1,12 +1,12 @@
 import os.path
+
 from lsst.obs.base.gen2to3 import ConvertRepoSkyMapConfig
 from lsst.obs.subaru import HyperSuprimeCam
 
 maskCollection = HyperSuprimeCam.makeCollectionName("masks")
 config.runsForced["brightObjectMask"] = maskCollection
 config.extraUmbrellaChildren.append(maskCollection)
 config.skyMaps["hsc_rings_v1"] = ConvertRepoSkyMapConfig()
-config.skyMaps["hsc_rings_v1"].load(os.path.join(os.path.dirname(__file__), "..", "makeSkyMap.py"))
 config.skyMaps["hsc_rings_v1"].load(os.path.join(os.path.dirname(__file__), "makeSkyMap.py"))
 # If there's no skymap in the root repo, but some dataset defined on
 # tracts/patches is present there (i.e. brightObjectMask), assume this

config/hsc/dark.py → config/dark.py

@@ -1,6 +1,5 @@
 import os.path
 
-
 config.isr.load(os.path.join(os.path.dirname(__file__), "isr.py"))
 
 config.isr.doBias = True

config/hsc/datasetIngest-gen3.py → config/datasetIngest-gen3.py

@@ -1,6 +1,6 @@
 # Config override for lsst.ap.verify.Gen3DatasetIngestTask
 
-import os
+import os.path
 
 configDir = os.path.dirname(__file__)
 

config/hsc/displayCatalog.py → config/displayCatalog.py

@@ -1,4 +1,3 @@
 import os.path
 
-
 config.load(os.path.join(os.path.dirname(__file__), "srcSchemaMap.py"))

config/hsc/fgcmBuildStars.py → config/fgcmBuildStars.py

@@ -1,10 +1,5 @@
-"""
-HSC-specific overrides for FgcmBuildStars
-"""
-
 import os.path
 
-
 # Demand at least 2 observations of a star to be considered for calibration
 config.minPerBand = 2
 # High density cut to keep high density regions from dominating statistics.
@@ -36,4 +31,3 @@
 config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.errField = 'i_fluxErr'
 # Minimum signal-to-noise cut for a reference star to be considered a match.
 config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.minimum = 10.0
-

config/hsc/fgcmBuildStarsTable.py → config/fgcmBuildStarsTable.py

@@ -1,10 +1,5 @@
-"""
-HSC-specific overrides for FgcmBuildStarsTable
-"""
-
 import os.path
 
-
 # Demand at least 2 observations of a star to be considered for calibration
 config.minPerBand = 2
 # High density cut to keep high density regions from dominating statistics.
@@ -36,4 +31,3 @@
 config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.errField = 'i_fluxErr'
 # Minimum signal-to-noise cut for a reference star to be considered a match.
 config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.minimum = 10.0
-

config/hsc/fgcmFitCycle.py → config/fgcmFitCycle.py

@@ -1,7 +1,3 @@
-"""
-HSC-specific overrides for FgcmFitCycle
-"""
-
 from lsst.fgcmcal import Sedterm, Sedboundaryterm
 
 config.outfileBase = 'fgcmHscCalibrations'
@@ -133,7 +129,6 @@
                               'y': 1.0,
                               'N1010': 1.0}
 
-from lsst.fgcmcal import Sedterm, Sedboundaryterm
 config.sedboundaryterms.data = {'gr': Sedboundaryterm(primary='g', secondary='r'),
                                 'ri': Sedboundaryterm(primary='r', secondary='i'),
                                 'iz': Sedboundaryterm(primary='i', secondary='z'),

config/hsc/fgcmMakeLut.py → config/fgcmMakeLut.py

@@ -1,7 +1,3 @@
-"""
-HSC-specific overrides for FgcmMakeLut
-"""
-
 # Filter names to build LUT ('short' names)
 config.filterNames = ('g', 'r', 'r2', 'i', 'i2', 'z', 'y',
                       'N387', 'N816', 'N921', 'N1010')

config/fgcmOutputProducts.py

@@ -0,0 +1 @@
+config.cycleNumber = 5

config/filterMap.py

@@ -1,2 +1,52 @@
-for source, target in [('B', 'g'), ('V', 'r'), ('R', 'r'), ('I', 'i')]:
+for source, target in [
+        # Generic names
+        ('B', 'g'),
+        ('V', 'r'),
+        ('R', 'r'),
+        ('I', 'i'),
+
+        # Names used by Exposure.getFilter() in Gen2.
+        # Wide bands
+        ("r2", "r"),
+        ("i2", "i"),
+        # Narrow bands
+        ('N387', 'g'),
+        ('N468', 'g'),
+        ('N515', 'g'),
+        ('N527', 'g'),
+        ('N656', 'r'),
+        ('N718', 'i'),
+        ('N816', 'i'),
+        ('N921', 'z'),
+        ('N926', 'z'),
+        ('N973', 'y'),
+        ('N1010', 'y'),
+        # Intermediate bands
+        ('I945', 'z'),
+
+        # Names used by data IDs in both Gen2 and Gen3, and
+        # Exposure.getFilter() in Gen3 (mappings are the same).
+        # Wide bands
+        ("HSC-G", "g"),
+        ("HSC-R", "r"),
+        ("HSC-R2", "r"),
+        ("HSC-I", "i"),
+        ("HSC-I2", "i"),
+        ("HSC-Z", "z"),
+        ("HSC-Y", "y"),
+        # Narrow bands
+        ('NB0387', 'g'),
+        ('NB0468', 'g'),
+        ('NB0515', 'g'),
+        ('NB0527', 'g'),
+        ('NB0656', 'r'),
+        ('NB0718', 'i'),
+        ('NB0816', 'i'),
+        ('NB0921', 'z'),
+        ('NB0926', 'z'),
+        ('NB0973', 'y'),
+        ('NB1010', 'y'),
+        # Intermediate bands
+        ('IB0945', 'z'),
+    ]:
     config.filterMap[source] = target

config/hsc/flat.py → config/flat.py

@@ -1,6 +1,5 @@
 import os.path
 
-
 config.isr.load(os.path.join(os.path.dirname(__file__), "isr.py"))
 
 from lsst.obs.hsc.calibs import HscFlatCombineTask

config/forcedPhotCcd.py

@@ -1,7 +1,10 @@
 import os.path
 
-
 config.measurement.load(os.path.join(os.path.dirname(__file__), "apertures.py"))
 config.measurement.load(os.path.join(os.path.dirname(__file__), "kron.py"))
 
 config.measurement.slots.gaussianFlux = None
+config.doApplyExternalPhotoCalib = True
+config.externalPhotoCalibName = 'fgcm'
+config.doApplyExternalSkyWcs = True
+config.doApplySkyCorr = True

config/fringe.py

@@ -1,7 +1,10 @@
-import os
+import os.path
 
 configDir = os.path.dirname(__file__)
 bgFile = os.path.join(configDir, "background.py")
 
 config.detection.background.load(bgFile)
 config.subtractBackground.load(bgFile)
+
+config.isr.load(os.path.join(configDir, "isr.py"))
+config.isr.doBrighterFatter = False