let ConsoleMonitor display only median and std values; fix bug with F…

…ileLogMonitor meta file location; fix MetricsProcessor events triggering; implement metrics calculation while training test

piepline/monitoring/monitors.py

@@ -7,7 +7,6 @@
 
 from piepline.utils.fsm import FolderRegistrable, FileStructManager
 from piepline.train_config.metrics import MetricsGroup, AbstractMetric
-from piepline.utils.utils import dict_recursive_bypass
 
 __all__ = ['AbstractMonitor', 'AbstractMetricsMonitor', 'ConsoleLossMonitor', 'FileLogMonitor']
 
@@ -107,11 +106,10 @@ def __str__(self):
 
     def update_losses(self, losses: {}) -> None:
         def on_loss(name: str, values: np.ndarray, string) -> None:
-            string.append(" {}: [{:4f}, {:4f}, {:4f}];".format(name, np.min(values), np.mean(values), np.max(values)))
+            string.append(" {}: [{:4f}, {:4f}];".format(name, np.median(values), np.std(values)))
 
         res_string = self.ResStr("Epoch: [{}];".format(self._epoch_num))
         self._iterate_by_losses(losses, lambda m, v: on_loss(m, v, res_string))
-        print(res_string)
 
 
 class FileLogMonitor(AbstractMetricsMonitor, AbstractLossMonitor, FolderRegistrable):
@@ -163,7 +161,7 @@ def _process_metric(self, path: List[MetricsGroup], metric: 'AbstractMetric'):
             self._files[cur_file_path] = {'name': metric.name(), 'path': [p.name() for p in path]}
 
             if self._meta_file is None:
-                self._meta_file = os.path.join(cur_dir, 'meta.json')
+                self._meta_file = os.path.join(file_log_dir, 'meta.json')
 
             with open(self._meta_file, 'w') as meta_out:
                 json.dump(self._files, meta_out)
@@ -185,6 +183,9 @@ def write_final_metrics(self, path: str = 'metrics.json') -> 'FileLogMonitor':
         self._final_file = path
         return self
 
+    def get_dir(self) -> str:
+        return self._fsm.get_path(self, create_if_non_exists=False, check=False)
+
     def _get_gir(self) -> str:
         return os.path.join('monitors', 'metrics_log')
 

piepline/train_config/metrics_processor.py

@@ -23,10 +23,6 @@ def subscribe_to_stage(self, stage: AbstractStage) -> 'MetricsProcessor':
         events_container.event(stage, 'STAGE_END').add_callback(lambda s: self.reset_metrics())
         return self
 
-    def subscribe_to_trainer(self, trainer: Trainer) -> 'MetricsProcessor':
-        events_container.event(trainer, 'EPOCH_END').add_callback(lambda s: self.reset_metrics())
-        return self
-
     def add_metric(self, metric: AbstractMetric) -> AbstractMetric:
         """
         Add :class:`AbstractMetric` object

tests/common.py

@@ -31,9 +31,16 @@ def res(*args, **kwargs):
 
 
 class SimpleMetric(AbstractMetric):
-    def __init__(self, name: str = None, coeff: float = 1):
+    def __init__(self, name: str = None, coeff: float = 1, collect_values: bool = False):
         super().__init__('SimpleMetric' if name is None else name)
         self._coeff = coeff
+        self._collect_values = collect_values
+        self._inputs = []
 
     def calc(self, output: Tensor, target: Tensor) -> np.ndarray or float:
-        return F.pairwise_distance(output, target, p=2).cpu().detach().numpy() * self._coeff
+        if self._collect_values:
+            if len(self._values) == 0:
+                self._inputs = []
+            self._inputs.append((output.clone(), target.clone()))
+        res = F.pairwise_distance(output, target, p=2).cpu().detach().numpy() * self._coeff
+        return res

tests/train_test.py

@@ -1,10 +1,13 @@
+import json
 import os
 from random import randint
 
 import torch
 import numpy as np
 from torch import Tensor
+from torch.nn import functional as F
 
+from monitoring.monitors import FileLogMonitor
 from piepline.train import Trainer
 from piepline import events_container
 from piepline.train_config.metrics import AbstractMetric, MetricsGroup
@@ -192,14 +195,22 @@ def test_metric_calc_in_train_loop(self):
             .set_epoch_num(2)
 
         mp = MetricsProcessor()
-        mp.add_metrics_group(MetricsGroup('grp1').add(SimpleMetric()))
-        mp.add_metrics_group(MetricsGroup('grp2').add(SimpleMetric()))
+        metric1 = SimpleMetric(coeff=1, collect_values=True)
+        # metric2 = SimpleMetric(coeff=1.7, collect_values=True)
+        mp.add_metrics_group(MetricsGroup('grp1').add(metric1))
+        # mp.add_metrics_group(MetricsGroup('grp2').add(metric2))
 
-        mp.subscribe_to_stage(stages[0]).subscribe_to_stage(stages[1])
-        mp.subscribe_to_trainer(trainer)
+        mp.subscribe_to_stage(stages[0])  # .subscribe_to_stage(stages[1])
+        # mp.subscribe_to_trainer(trainer)
 
-        mh = MonitorHub(trainer).subscribe2metrics_processor(mp)
-
-        # TODO: continue implementing test
+        file_monitor_hub = FileLogMonitor(fsm).write_final_metrics()
+        MonitorHub(trainer).subscribe2metrics_processor(mp).add_monitor(file_monitor_hub)
 
         trainer.train()
+
+        with open(os.path.join(file_monitor_hub.get_dir(), 'metrics.json'), 'r') as metrics_file:
+            metrics = json.load(metrics_file)
+
+        self.assertAlmostEqual(metrics['grp1/SimpleMetric'],
+                               float(np.mean([F.pairwise_distance(i[0], i[1], p=2).cpu().detach().numpy() for i in metric1._inputs])),
+                               delta=1e-2)