Merge branch 'master' into tickets/DM-26757

python/lsst/meas/algorithms/defects.py

@@ -532,14 +532,19 @@ def _get_values(values, n=1):
         return values[:n]
 
     @classmethod
-    def fromTable(cls, table):
+    def fromTable(cls, table, normalize_on_init=True):
         """Construct a `Defects` from the contents of a
         `~lsst.afw.table.BaseCatalog`.
 
         Parameters
         ----------
         table : `lsst.afw.table.BaseCatalog`
             Table with one row per defect.
+        normalize_on_init : `bool`, optional
+            If `True`, normalization is applied to the defects listed in the
+            table to remove duplicates, eliminate overlaps, etc. Otherwise
+            the defects in the returned object exactly match those in the
+            table.
 
         Returns
         -------
@@ -635,7 +640,7 @@ def fromTable(cls, table):
 
             defectList.append(box)
 
-        defects = cls(defectList)
+        defects = cls(defectList, normalize_on_init=normalize_on_init)
         defects.setMetadata(table.getMetadata())
 
         # Once read, the schema headers are irrelevant
@@ -647,31 +652,41 @@ def fromTable(cls, table):
         return defects
 
     @classmethod
-    def readFits(cls, *args):
+    def readFits(cls, *args, normalize_on_init=False):
         """Read defect list from FITS table.
 
         Parameters
         ----------
         *args
             Arguments to be forwarded to
-            `lsst.afw.table.BaseCatalog.writeFits`.
+            `lsst.afw.table.BaseCatalog.readFits`.
+        normalize_on_init : `bool`, optional
+            If `True`, normalization is applied to the defects read fom the
+            file to remove duplicates, eliminate overlaps, etc. Otherwise
+            the defects in the returned object exactly match those in the
+            file.
 
         Returns
         -------
         defects : `Defects`
             Defects read from a FITS table.
         """
         table = lsst.afw.table.BaseCatalog.readFits(*args)
-        return cls.fromTable(table)
+        return cls.fromTable(table, normalize_on_init=normalize_on_init)
 
     @classmethod
-    def readText(cls, filename):
+    def readText(cls, filename, normalize_on_init=False):
         """Read defect list from standard format text table file.
 
         Parameters
         ----------
         filename : `str`
             Name of the file containing the defects definitions.
+        normalize_on_init : `bool`, optional
+            If `True`, normalization is applied to the defects read fom the
+            file to remove duplicates, eliminate overlaps, etc. Otherwise
+            the defects in the returned object exactly match those in the
+            file.
 
         Returns
         -------
@@ -706,16 +721,21 @@ def readText(cls, filename):
         afwTable.setMetadata(metadata)
 
         # Extract defect information from the table itself
-        return cls.fromTable(afwTable)
+        return cls.fromTable(afwTable, normalize_on_init=normalize_on_init)
 
     @classmethod
-    def readLsstDefectsFile(cls, filename):
+    def readLsstDefectsFile(cls, filename, normalize_on_init=False):
         """Read defects information from a legacy LSST format text file.
 
         Parameters
         ----------
         filename : `str`
             Name of text file containing the defect information.
+        normalize_on_init : `bool`, optional
+            If `True`, normalization is applied to the defects read fom the
+            file to remove duplicates, eliminate overlaps, etc. Otherwise
+            the defects in the returned object exactly match those in the
+            file.
 
         Returns
         -------
@@ -747,10 +767,12 @@ def readLsstDefectsFile(cls, filename):
                                   dtype=[("x0", "int"), ("y0", "int"),
                                          ("x_extent", "int"), ("y_extent", "int")])
 
-        return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
+        defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
                                    lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
                    for row in defect_array)
 
+        return cls(defects, normalize_on_init=normalize_on_init)
+
     @classmethod
     def fromFootprintList(cls, fpList):
         """Compute a defect list from a footprint list, optionally growing

python/lsst/meas/algorithms/detection.py

@@ -453,7 +453,7 @@ def convolveImage(self, maskedImage, psf, doSmooth=True):
         self.metadata.set("sigma", sigma)
 
         if not doSmooth:
-            middle = maskedImage.Factory(maskedImage)
+            middle = maskedImage.Factory(maskedImage, deep=True)
             return pipeBase.Struct(middle=middle, sigma=sigma)
 
         # Smooth using a Gaussian (which is separable, hence fast) of width sigma

python/lsst/meas/algorithms/loadReferenceObjects.py

@@ -691,20 +691,23 @@ def remapReferenceCatalogSchema(refCat, *, filterNameList=None, position=False,
 def getRefFluxField(schema, filterName=None):
     """Get the name of a flux field from a schema.
 
-    if filterName is specified:
-        return *filterName*_camFlux if present
-        else return *filterName*_flux if present (camera filter name matches reference filter name)
+    return the alias of "anyFilterMapsToThis", if present
+    else if filterName is specified:
+        return "*filterName*_camFlux" if present
+        else return "*filterName*_flux" if present (camera filter name
+            matches reference filter name)
         else throw RuntimeError
     else:
-        return camFlux, if present,
+        return "camFlux", if present,
         else throw RuntimeError
 
     Parameters
     ----------
     schema : `lsst.afw.table.Schema`
         Reference catalog schema.
-    filterName : `str`
-        Name of camera filter.
+    filterName : `str`, optional
+        Name of camera filter. If not specified, ``defaultFilter`` needs to be
+        set in the refcat loader config.
 
     Returns
     -------
@@ -718,6 +721,11 @@ def getRefFluxField(schema, filterName=None):
     """
     if not isinstance(schema, afwTable.Schema):
         raise RuntimeError("schema=%s is not a schema" % (schema,))
+    try:
+        return schema.getAliasMap().get("anyFilterMapsToThis")
+    except LookupError:
+        pass  # try the filterMap next
+
     if filterName:
         fluxFieldList = [filterName + "_camFlux", filterName + "_flux"]
     else:
@@ -770,14 +778,25 @@ class LoadReferenceObjectsConfig(pexConfig.Config):
         min=0,
     )
     defaultFilter = pexConfig.Field(
-        doc="Default reference catalog filter to use if filter not specified in exposure; "
-        "if blank then filter must be specified in exposure",
+        doc=("Default reference catalog filter to use if filter not specified in exposure;"
+             " if blank then filter must be specified in exposure."),
         dtype=str,
         default="",
     )
+    anyFilterMapsToThis = pexConfig.Field(
+        doc=("Always use this reference catalog filter, no matter whether or what filter name is "
+             "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
+             " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
+             "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
+             "photometry, which need a filterMap and/or colorterms/transmission corrections."),
+        dtype=str,
+        default=None,
+        optional=True
+    )
     filterMap = pexConfig.DictField(
-        doc="Mapping of camera filter name: reference catalog filter name; "
-        "each reference filter must exist",
+        doc=("Mapping of camera filter name: reference catalog filter name; "
+             "each reference filter must exist in the refcat."
+             " Note that this does not perform any bandpass corrections: it is just a lookup."),
         keytype=str,
         itemtype=str,
         default={},
@@ -789,6 +808,14 @@ class LoadReferenceObjectsConfig(pexConfig.Config):
         default=False,
     )
 
+    def validate(self):
+        super().validate()
+        if self.filterMap != {} and self.anyFilterMapsToThis is not None:
+            msg = "`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
+            raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
+                                                 self, msg)
+
+
 # The following comment block adds a link to this task from the Task Documentation page.
 ## @addtogroup LSST_task_documentation
 ## @{
@@ -1046,13 +1073,21 @@ def _addFluxAliases(self, schema):
         schema : `lsst.afw.table.Schema`
             Schema for reference catalog.
 
-        Throws
+        Raises
         ------
         RuntimeError
             If any reference flux field is missing from the schema.
         """
         aliasMap = schema.getAliasMap()
 
+        if self.config.anyFilterMapsToThis is not None:
+            refFluxName = self.config.anyFilterMapsToThis + "_flux"
+            if refFluxName not in schema:
+                msg = f"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
+                raise RuntimeError(msg)
+            aliasMap.set("anyFilterMapsToThis", refFluxName)
+            return  # this is mutually exclusive with filterMap
+
         def addAliasesForOneFilter(filterName, refFilterName):
             """Add aliases for a single filter
 

tests/test_interp.py

@@ -140,7 +140,8 @@ def testAstropyRegion(self):
         # The two coincident points are combined on read, so we end up with two defects.
 
         with self.assertLogs():
-            defects = algorithms.Defects.readFits(os.path.join(TESTDIR, "data", "fits_region.fits"))
+            defects = algorithms.Defects.readFits(os.path.join(TESTDIR, "data", "fits_region.fits"),
+                                                  normalize_on_init=True)
 
         self.assertEqual(len(defects), 2)
 
@@ -161,7 +162,7 @@ def testLsstTextfile(self):
 10 10 10 10
 """, file=fh)
 
-            defects = algorithms.Defects.readLsstDefectsFile(tmpFile)
+            defects = algorithms.Defects.readLsstDefectsFile(tmpFile, normalize_on_init=True)
 
         # Although there are 9 defects listed above, we record 11 after
         # normalization. This is due to non-optimal behaviour in

tests/test_loadReferenceObjects.py

@@ -28,6 +28,7 @@
 import lsst.log
 from lsst.meas.algorithms import LoadReferenceObjectsTask, getRefFluxField, getRefFluxKeys
 from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits, convertToNanojansky
+import lsst.pex.config
 import lsst.utils.tests
 
 
@@ -45,6 +46,27 @@ class TestLoadReferenceObjects(lsst.utils.tests.TestCase):
     Only methods with concrete implementations are tested (hence not loadSkyCircle)
     """
 
+    def testFilterMapVsAnyFilterMapsToThis(self):
+        config = TrivialLoader.ConfigClass()
+        # check that a filterMap-only config passes validation
+        config.filterMap = {"b": "a"}
+        try:
+            config.validate()
+        except lsst.pex.config.FieldValidationError:
+            self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.")
+
+        # anyFilterMapsToThis and filterMap are mutually exclusive
+        config.anyFilterMapsToThis = "c"
+        with self.assertRaises(lsst.pex.config.FieldValidationError):
+            config.validate()
+
+        # check that a anyFilterMapsToThis-only config passes validation
+        config.filterMap = {}
+        try:
+            config.validate()
+        except lsst.pex.config.FieldValidationError:
+            self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.")
+
     def testMakeMinimalSchema(self):
         """Make a schema and check it."""
         for filterNameList in (["r"], ["foo", "_bar"]):
@@ -153,6 +175,19 @@ def testFilterAliasMap(self):
                     with self.assertRaises(RuntimeError):
                         getRefFluxKeys(refSchema, "camr")
 
+    def testAnyFilterMapsToThisAlias(self):
+        # test anyFilterMapsToThis
+        config = TrivialLoader.ConfigClass()
+        config.anyFilterMapsToThis = "gg"
+        loader = TrivialLoader(config=config)
+        refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["gg"])
+        loader._addFluxAliases(refSchema)
+        self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux")
+        # raise if "gg" is not in the refcat filter list
+        with self.assertRaises(RuntimeError):
+            refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["rr"])
+            refSchema = loader._addFluxAliases(refSchema)
+
     def testCheckFluxUnits(self):
         """Test that we can identify old style fluxes in a schema."""
         schema = LoadReferenceObjectsTask.makeMinimalSchema(['r', 'z'])