Change the name of base evaluator class from BaseEvaluator to Evaluat…

…or (#101)

* delete the old Evaluator and change name of baseEvaluator to Evaluator

* change import path for places that imported the baseevaluator

* put OldEvaluator back in

* fix up unit tests

---------

Co-authored-by: Eric Charles <badass@stanford.edu>

src/rail/evaluation/dist_to_dist_evaluator.py

@@ -3,14 +3,14 @@
 from qp.metrics.concrete_metric_classes import DistToDistMetric
 
 from rail.core.data import QPHandle
-from rail.evaluation.evaluator import BaseEvaluator
+from rail.evaluation.evaluator import Evaluator
 
 
-class DistToDistEvaluator(BaseEvaluator):
+class DistToDistEvaluator(Evaluator):
     """Evaluate the performance of a photo-z estimator against reference PDFs"""
 
     name = "DistToDistEvaluator"
-    config_options = BaseEvaluator.config_options.copy()
+    config_options = Evaluator.config_options.copy()
     config_options.update(
         limits=Param(
             tuple,

src/rail/evaluation/dist_to_point_evaluator.py

@@ -3,14 +3,14 @@
 from qp.metrics.concrete_metric_classes import DistToPointMetric
 
 from rail.core.data import QPHandle, TableHandle
-from rail.evaluation.evaluator import BaseEvaluator
+from rail.evaluation.evaluator import Evaluator
 
 
-class DistToPointEvaluator(BaseEvaluator):
+class DistToPointEvaluator(Evaluator):
     """Evaluate the performance of a photo-z estimator against reference point estimate"""
 
     name = "DistToPointEvaluator"
-    config_options = BaseEvaluator.config_options.copy()
+    config_options = Evaluator.config_options.copy()
     config_options.update(
         limits=Param(
             tuple,

src/rail/evaluation/evaluator.py

@@ -21,147 +21,6 @@
     PointSigmaMAD,
 )
 
-
-class Evaluator(RailStage):
-    """Evaluate the performance of a photo-Z estimator"""
-
-    name = "Evaluator"
-    config_options = RailStage.config_options.copy()
-    config_options.update(
-        zmin=Param(float, 0., msg="min z for grid"),
-        zmax=Param(float, 3.0, msg="max z for grid"),
-        nzbins=Param(int, 301, msg="# of bins in zgrid"),
-        pit_metrics=Param(str, 'all', msg='PIT-based metrics to include'),
-        point_metrics=Param(str, 'all', msg='Point-estimate metrics to include'),
-        hdf5_groupname=Param(str, '', msg='Name of group in hdf5 where redshift data is located'),
-        do_cde=Param(bool, True, msg='Evaluate CDE Metric'),
-        redshift_col=SHARED_PARAMS,
-    )
-    inputs = [('input', QPHandle),
-              ('truth', Hdf5Handle)]
-    outputs = [("output", Hdf5Handle)]
-
-    def __init__(self, args, comm=None):
-        """Initialize Evaluator"""
-        RailStage.__init__(self, args, comm=comm)
-
-    def evaluate(self, data, truth):
-        """Evaluate the performance of an estimator
-
-        This will attach the input data and truth to this `Evaluator`
-        (for introspection and provenance tracking).
-
-        Then it will call the run() and finalize() methods, which need to
-        be implemented by the sub-classes.
-
-        The run() method will need to register the data that it creates to this Estimator
-        by using `self.add_data('output', output_data)`.
-
-        Parameters
-        ----------
-        data : qp.Ensemble
-            The sample to evaluate
-        truth : Table-like
-            Table with the truth information
-
-        Returns
-        -------
-        output : Table-like
-            The evaluation metrics
-        """
-
-        self.set_data("input", data)
-        self.set_data("truth", truth)
-        self.run()
-        self.finalize()
-        return self.get_handle("output")
-
-    def run(self):
-        """Run method
-
-        Evaluate all the metrics and put them into a table
-
-        Notes
-        -----
-        Get the input data from the data store under this stages 'input' tag
-        Get the truth data from the data store under this stages 'truth' tag
-        Puts the data into the data store under this stages 'output' tag
-        """
-
-        pz_data = self.get_data('input')
-        if self.config.hdf5_groupname:  # pragma: no cover
-            specz_data = self.get_data('truth')[self.config.hdf5_groupname]
-        else: 
-            specz_data = self.get_data('truth')
-        z_true = specz_data[self.config['redshift_col']]
-
-        zgrid = np.linspace(self.config.zmin, self.config.zmax, self.config.nzbins+1)
-
-        # Create an instance of the PIT class
-        pitobj = PIT(pz_data, z_true)
-
-        # Build reference dictionary of the PIT meta-metrics from this PIT instance
-        PIT_METRICS = dict(
-            AD=getattr(pitobj, "evaluate_PIT_anderson_ksamp"),
-            CvM=getattr(pitobj, "evaluate_PIT_CvM"),
-            KS=getattr(pitobj, "evaluate_PIT_KS"),
-            OutRate=getattr(pitobj, "evaluate_PIT_outlier_rate"),
-        )
-
-        # Parse the input configuration to determine which meta-metrics should be calculated
-        if self.config.pit_metrics == "all":
-            pit_metrics = list(PIT_METRICS.keys())
-        else:  # pragma: no cover
-            pit_metrics = self.config.pit_metrics.split()
-
-        # Evaluate each of the requested meta-metrics, and store the result in `out_table`
-        out_table = {}
-        for pit_metric in pit_metrics:
-            value = PIT_METRICS[pit_metric]()
-
-            # The result objects of some meta-metrics are bespoke scipy objects with inconsistent fields.
-            # Here we do our best to store the relevant fields in `out_table`.
-            if isinstance(value, list):  # pragma: no cover
-                out_table[f"PIT_{pit_metric}"] = value
-            else:
-                out_table[f"PIT_{pit_metric}_stat"] = [
-                    getattr(value, "statistic", None)
-                ]
-                out_table[f"PIT_{pit_metric}_pval"] = [getattr(value, "p_value", None)]
-                out_table[f"PIT_{pit_metric}_significance_level"] = [
-                    getattr(value, "significance_level", None)
-                ]
-
-        POINT_METRICS = dict(
-            SimgaIQR=PointSigmaIQR,
-            Bias=PointBias,
-            OutlierRate=PointOutlierRate,
-            SigmaMAD=PointSigmaMAD,
-        )
-        if self.config.point_metrics == "all":
-            point_metrics = list(POINT_METRICS.keys())
-        else:  # pragma: no cover
-            point_metrics = self.config.point_metrics.split()
-
-        z_mode = None
-        for point_metric in point_metrics:
-            if z_mode is None:
-                z_mode = np.squeeze(pz_data.mode(grid=zgrid))
-            value = POINT_METRICS[point_metric](z_mode, z_true).evaluate()
-            out_table[f"POINT_{point_metric}"] = [value]
-
-        if self.config.do_cde:
-            value = CDELoss(pz_data, zgrid, z_true).evaluate()
-            out_table["CDE_stat"] = [value.statistic]
-            out_table["CDE_pval"] = [value.p_value]
-
-        # Converting any possible None to NaN to write it
-        out_table_to_write = {
-            key: np.array(val).astype(float) for key, val in out_table.items()
-        }
-        self.add_data("output", out_table_to_write)
-
-
 def _all_subclasses(a_class):
     return set(a_class.__subclasses__()).union(
         [s for c in a_class.__subclasses__() for s in _all_subclasses(c)]
@@ -177,10 +36,10 @@ def _build_metric_dict(a_class):
     return the_dict
 
 
-class BaseEvaluator(RailStage):
+class Evaluator(RailStage):
     """Evaluate the performance of a photo-z estimator against reference point estimate"""
 
-    name = "BaseEvaluator"
+    name = "Evaluator"
     config_options = RailStage.config_options.copy()
     config_options.update(
         metrics=Param(
@@ -345,7 +204,7 @@ def _get_all_data(self):
         return all_data
 
     def _process_chunk(self, data_tuple, first):  # pragma: no cover
-        raise NotImplementedError("BaseEvaluator._process_chunk()")
+        raise NotImplementedError("Evaluator._process_chunk()")
 
     def _process_all_chunk_metrics(self, estimate_data, reference_data, start, end, first):
         """This function takes the properly formatted data and loops over all the
@@ -418,7 +277,7 @@ def _output_table_chunk_data(self, start, end, out_table, first):
 
 
     def _process_all(self, data_tuple):  # pragma: no cover
-        raise NotImplementedError("BaseEvaluator._process_all()")
+        raise NotImplementedError("Evaluator._process_all()")
 
 
     def _process_all_metrics(self, estimate_data, reference_data):
@@ -489,3 +348,136 @@ def _build_config_dict(self):
             this_metric_class = self._metric_dict[metric_name_]
             this_metric = this_metric_class(**sub_dict)
             self._cached_metrics[metric_name_] = this_metric
+
+
+class OldEvaluator(RailStage):
+    """Evaluate the performance of a photo-Z estimator"""
+
+    name = "OldEvaluator"
+    config_options = RailStage.config_options.copy()
+    config_options.update(
+        zmin=Param(float, 0., msg="min z for grid"),
+        zmax=Param(float, 3.0, msg="max z for grid"),
+        nzbins=Param(int, 301, msg="# of bins in zgrid"),
+        pit_metrics=Param(str, 'all', msg='PIT-based metrics to include'),
+        point_metrics=Param(str, 'all', msg='Point-estimate metrics to include'),
+        hdf5_groupname=Param(str, '', msg='Name of group in hdf5 where redshift data is located'),
+        do_cde=Param(bool, True, msg='Evaluate CDE Metric'),
+        redshift_col=SHARED_PARAMS,
+    )
+    inputs = [('input', QPHandle),
+              ('truth', Hdf5Handle)]
+    outputs = [("output", Hdf5Handle)]
+
+    def __init__(self, args, comm=None):
+        """Initialize Evaluator"""
+        RailStage.__init__(self, args, comm=comm)
+
+    def evaluate(self, data, truth):
+        """Evaluate the performance of an estimator
+        This will attach the input data and truth to this `Evaluator`
+        (for introspection and provenance tracking).
+        Then it will call the run() and finalize() methods, which need to
+        be implemented by the sub-classes.
+        The run() method will need to register the data that it creates to this Estimator
+        by using `self.add_data('output', output_data)`.
+        Parameters
+        ----------
+        data : qp.Ensemble
+            The sample to evaluate
+        truth : Table-like
+            Table with the truth information
+        Returns
+        -------
+        output : Table-like
+            The evaluation metrics
+        """
+
+        self.set_data("input", data)
+        self.set_data("truth", truth)
+        self.run()
+        self.finalize()
+        return self.get_handle("output")
+
+    def run(self):
+        """Run method
+        Evaluate all the metrics and put them into a table
+        Notes
+        -----
+        Get the input data from the data store under this stages 'input' tag
+        Get the truth data from the data store under this stages 'truth' tag
+        Puts the data into the data store under this stages 'output' tag
+        """
+
+        pz_data = self.get_data('input')
+        if self.config.hdf5_groupname:  # pragma: no cover
+            specz_data = self.get_data('truth')[self.config.hdf5_groupname]
+        else: 
+            specz_data = self.get_data('truth')
+        z_true = specz_data[self.config['redshift_col']]
+
+        zgrid = np.linspace(self.config.zmin, self.config.zmax, self.config.nzbins+1)
+
+        # Create an instance of the PIT class
+        pitobj = PIT(pz_data, z_true)
+
+        # Build reference dictionary of the PIT meta-metrics from this PIT instance
+        PIT_METRICS = dict(
+            AD=getattr(pitobj, "evaluate_PIT_anderson_ksamp"),
+            CvM=getattr(pitobj, "evaluate_PIT_CvM"),
+            KS=getattr(pitobj, "evaluate_PIT_KS"),
+            OutRate=getattr(pitobj, "evaluate_PIT_outlier_rate"),
+        )
+
+        # Parse the input configuration to determine which meta-metrics should be calculated
+        if self.config.pit_metrics == "all":
+            pit_metrics = list(PIT_METRICS.keys())
+        else:  # pragma: no cover
+            pit_metrics = self.config.pit_metrics.split()
+
+        # Evaluate each of the requested meta-metrics, and store the result in `out_table`
+        out_table = {}
+        for pit_metric in pit_metrics:
+            value = PIT_METRICS[pit_metric]()
+
+            # The result objects of some meta-metrics are bespoke scipy objects with inconsistent fields.
+            # Here we do our best to store the relevant fields in `out_table`.
+            if isinstance(value, list):  # pragma: no cover
+                out_table[f"PIT_{pit_metric}"] = value
+            else:
+                out_table[f"PIT_{pit_metric}_stat"] = [
+                    getattr(value, "statistic", None)
+                ]
+                out_table[f"PIT_{pit_metric}_pval"] = [getattr(value, "p_value", None)]
+                out_table[f"PIT_{pit_metric}_significance_level"] = [
+                    getattr(value, "significance_level", None)
+                ]
+
+        POINT_METRICS = dict(
+            SimgaIQR=PointSigmaIQR,
+            Bias=PointBias,
+            OutlierRate=PointOutlierRate,
+            SigmaMAD=PointSigmaMAD,
+        )
+        if self.config.point_metrics == "all":
+            point_metrics = list(POINT_METRICS.keys())
+        else:  # pragma: no cover
+            point_metrics = self.config.point_metrics.split()
+
+        z_mode = None
+        for point_metric in point_metrics:
+            if z_mode is None:
+                z_mode = np.squeeze(pz_data.mode(grid=zgrid))
+            value = POINT_METRICS[point_metric](z_mode, z_true).evaluate()
+            out_table[f"POINT_{point_metric}"] = [value]
+
+        if self.config.do_cde:
+            value = CDELoss(pz_data, zgrid, z_true).evaluate()
+            out_table["CDE_stat"] = [value.statistic]
+            out_table["CDE_pval"] = [value.p_value]
+
+        # Converting any possible None to NaN to write it
+        out_table_to_write = {
+            key: np.array(val).astype(float) for key, val in out_table.items()
+        }
+        self.add_data("output", out_table_to_write)

src/rail/evaluation/point_to_point_evaluator.py

@@ -3,14 +3,14 @@
 from qp.metrics.point_estimate_metric_classes import PointToPointMetric
 
 from rail.core.data import TableHandle, QPHandle
-from rail.evaluation.evaluator import BaseEvaluator
+from rail.evaluation.evaluator import Evaluator
 
 
-class PointToPointEvaluator(BaseEvaluator):
+class PointToPointEvaluator(Evaluator):
     """Evaluate the performance of a photo-z estimator against reference point estimate"""
 
     name = "PointToPointEvaluator"
-    config_options = BaseEvaluator.config_options.copy()
+    config_options = Evaluator.config_options.copy()
     config_options.update(
         hdf5_groupname=Param(
             str, "photometry", required=False, msg="HDF5 Groupname for truth table."