DM-430: Modify measurement framework to log plugins error in individu…

…al logs

Logs named for the taskName + pluginName (e.g., measurement.base_PsfFlux)
Name is set by the task, but the name is pushed down to the plugin.
if the plugin class has "hasLogName = True" and an optional "logName"
parameter in its initialization.  If both are so, plugin.getLogName() wil
return the name of the log.

if lsst.log.Log logs are identified by name, not by the logger itself, so the
logName is enough.

C++ routines can use the BaseAlgorithm function getLogName to get the right log.

Log Measurement Errors, which were not previously logged.

Checkin seg fault problem with pybind methods on BaseAlgorithm
Add py:multiple_inheritance to SimpleAlgorithm wrapper.

include/lsst/meas/base/Algorithm.h

@@ -24,6 +24,8 @@
 #ifndef LSST_MEAS_BASE_Algorithm_h_INCLUDED
 #define LSST_MEAS_BASE_Algorithm_h_INCLUDED
 
+#include "lsst/log/Log.h"
+
 #include "lsst/afw/table/fwd.h"
 #include "lsst/afw/image/Exposure.h"
 #include "lsst/meas/base/exceptions.h"
@@ -62,6 +64,11 @@ class BaseAlgorithm {
 
     virtual ~BaseAlgorithm() {}
 
+    std::string getLogName() const {
+        return _logName;
+    }
+protected:
+    std::string _logName;
 };
 
 /**

include/lsst/meas/base/PsfFlux.h

@@ -78,7 +78,8 @@ class PsfFluxAlgorithm : public SimpleAlgorithm {
     /// The control object contains the configuration parameters for this algorithm.
     typedef PsfFluxControl Control;
 
-    PsfFluxAlgorithm(Control const & ctrl, std::string const & name, afw::table::Schema & schema);
+    PsfFluxAlgorithm(Control const & ctrl, std::string const & name, afw::table::Schema & schema,
+                     std::string const & logName = "");
 
     virtual void measure(
         afw::table::SourceRecord & measRecord,

python/lsst/meas/base/algorithm.cc

@@ -41,16 +41,19 @@ PYBIND11_PLUGIN(algorithm) {
     py::module mod("algorithm");
 
     /* Module level */
-    py::class_<SingleFrameAlgorithm, std::shared_ptr<SingleFrameAlgorithm>> clsSingleFrameAlgorithm(
+    py::class_<BaseAlgorithm, std::shared_ptr<BaseAlgorithm>> clsBaseAlgorithm(
+            mod, "BaseAlgorithm");
+    py::class_<SingleFrameAlgorithm, std::shared_ptr<SingleFrameAlgorithm>, BaseAlgorithm> clsSingleFrameAlgorithm(
             mod, "SingleFrameAlgorithm");
     py::class_<SimpleAlgorithm, std::shared_ptr<SimpleAlgorithm>, SingleFrameAlgorithm> clsSimpleAlgorithm(
-            mod, "SimpleAlgorithm");
+            mod, "SimpleAlgorithm", py::multiple_inheritance());
 
     /* Members */
     python::declareAlgorithm<SingleFrameAlgorithm>(clsSingleFrameAlgorithm);
 
     clsSimpleAlgorithm.def("measureForced", &SimpleAlgorithm::measureForced,
         "measRecord"_a, "exposure"_a, "refRecord"_a, "refWcs"_a);
+    clsBaseAlgorithm.def("getLogName", &SimpleAlgorithm::getLogName);
 
     return mod.ptr();
 }

python/lsst/meas/base/baseMeasurement.py

@@ -22,6 +22,7 @@
 #
 """Base measurement task, which subclassed by the single frame and forced measurement tasks.
 """
+from lsst.log import Log
 import lsst.pipe.base
 import lsst.pex.config
 
@@ -207,6 +208,9 @@ def __init__(self, algMetadata=None, **kwds):
             algMetadata = lsst.daf.base.PropertyList()
         self.algMetadata = algMetadata
 
+    def getPluginLogName(self, pluginName):
+        return self.log.getName() + '.' + pluginName
+
     def initializePlugins(self, **kwds):
         """Initialize the plugins (and slots) according to the configuration.
 
@@ -226,7 +230,14 @@ def initializePlugins(self, **kwds):
             self.plugins[self.config.slots.centroid] = None
         # Init the plugins, sorted by execution order.  At the same time add to the schema
         for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
-            self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
+            #   Pass logName to the plugin if the plugin is marked as using it
+            #   The task will use this name to log plugin errors, regardless.
+            if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName:
+                self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
+                   logName=self.getPluginLogName(name), **kwds)
+            else:
+                self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
+
         # In rare circumstances (usually tests), the centroid slot not be coming from an algorithm,
         # which means we'll have added something we don't want to the plugins map, and we should
         # remove it.
@@ -288,9 +299,13 @@ def doMeasurement(self, plugin, measRecord, *args, **kwds):
         except FATAL_EXCEPTIONS:
             raise
         except MeasurementError as error:
+            lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).warn(
+                          "MeasurementError in %s.measure on record %s: %s"
+                          % (plugin.name, measRecord.getId(), error))
             plugin.fail(measRecord, error)
         except Exception as error:
-            self.log.warn("Error in %s.measure on record %s: %s"
+            lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).warn(
+                          "Exception in %s.measure on record %s: %s"
                           % (plugin.name, measRecord.getId(), error))
             plugin.fail(measRecord)
 
@@ -349,11 +364,16 @@ def doMeasurementN(self, plugin, measCat, *args, **kwds):
             plugin.measureN(measCat, *args, **kwds)
         except FATAL_EXCEPTIONS:
             raise
+
         except MeasurementError as error:
             for measRecord in measCat:
+                lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).warn(
+                          "MeasurementError in %s.measureN on records %s-%s: %s"
+                          % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
                 plugin.fail(measRecord, error)
         except Exception as error:
             for measRecord in measCat:
                 plugin.fail(measRecord)
-            self.log.warn("Error in %s.measureN on records %s-%s: %s"
+                lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).warn(
+                          "Exception in %s.measureN on records %s-%s: %s"
                           % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))

python/lsst/meas/base/forcedMeasurement.py

@@ -72,7 +72,7 @@ class ForcedPlugin(BaseMeasurementPlugin):
 
     ConfigClass = ForcedPluginConfig
 
-    def __init__(self, config, name, schemaMapper, metadata):
+    def __init__(self, config, name, schemaMapper, metadata, logName=None):
         """Initialize the measurement object.
 
         @param[in]  config       An instance of this class's ConfigClass.
@@ -84,7 +84,7 @@ def __init__(self, config, name, schemaMapper, metadata):
                                       will be transferred before any plugins are run.
         @param[in]  metadata     Plugin metadata that will be attached to the output catalog
         """
-        BaseMeasurementPlugin.__init__(self, config, name)
+        BaseMeasurementPlugin.__init__(self, config, name, logName=logName)
 
     def measure(self, measRecord, exposure, refRecord, refWcs):
         """Measure the properties of a source on a single image, given data from a

python/lsst/meas/base/plugins.py

@@ -68,7 +68,7 @@
 
 wrapSimpleAlgorithm(PsfFluxAlgorithm, Control=PsfFluxControl,
                     TransformClass=PsfFluxTransform, executionOrder=BasePlugin.FLUX_ORDER,
-                    shouldApCorr=True)
+                    shouldApCorr=True, hasLogName=True)
 wrapSimpleAlgorithm(PeakLikelihoodFluxAlgorithm, Control=PeakLikelihoodFluxControl,
                     TransformClass=PeakLikelihoodFluxTransform, executionOrder=BasePlugin.FLUX_ORDER)
 wrapSimpleAlgorithm(GaussianFluxAlgorithm, Control=GaussianFluxControl,

python/lsst/meas/base/pluginsBase.py

@@ -77,7 +77,7 @@ def getExecutionOrder(cls):
         """
         raise NotImplementedError("All plugins must implement getExecutionOrder()")
 
-    def __init__(self, config, name):
+    def __init__(self, config, name, logName=None):
         """!
         Initialize the plugin object.
 
@@ -87,6 +87,10 @@ def __init__(self, config, name):
         object.__init__(self)
         self.config = config
         self.name = name
+	self.logName = logName
+
+    def getLogName(self):
+        return self.logName
 
     def fail(self, measRecord, error=None):
         """!

python/lsst/meas/base/psfFlux.cc

@@ -64,6 +64,9 @@ PyFluxAlgorithm declareFluxAlgorithm(py::module &mod) {
     cls.def(py::init<PsfFluxAlgorithm::Control const &, std::string const &, afw::table::Schema &>(),
             "ctrl"_a, "name"_a, "schema"_a);
 
+    cls.def(py::init<PsfFluxAlgorithm::Control const &, std::string const &, afw::table::Schema &,
+            std::string const &>(),
+            "ctrl"_a, "name"_a, "schema"_a, "logName"_a);
     return cls;
 }
 

python/lsst/meas/base/sfm.py

@@ -61,7 +61,7 @@ class SingleFramePlugin(BaseMeasurementPlugin):
     registry = PluginRegistry(SingleFramePluginConfig)
     ConfigClass = SingleFramePluginConfig
 
-    def __init__(self, config, name, schema, metadata):
+    def __init__(self, config, name, schema, metadata, logName=None, **kwds):
         """!
         Initialize the measurement object.
 
@@ -70,8 +70,9 @@ def __init__(self, config, name, schema, metadata):
         @param[in,out]  schema   The Source schema.  New fields should be added here to
                                  hold measurements produced by this plugin.
         @param[in]  metadata     Plugin metadata that will be attached to the output catalog
+        @param[in]  logName      May include logName to use for this plugin
         """
-        BaseMeasurementPlugin.__init__(self, config, name)
+        BaseMeasurementPlugin.__init__(self, config, name, logName=logName)
 
     def measure(self, measRecord, exposure):
         """!