clean-way-to-support-ce

fluid/mnist/.run_ce.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# This file is only used for continuous evaluation.
+
+rm -rf *_factor.txt
+model_file='model.py'
+python $model_file --batch_size 128 --pass_num 5 --device CPU | python _ce.py

fluid/mnist/_ce.py

@@ -0,0 +1,61 @@
+# this file is only used for continuous evaluation test!
+
+import os
+import sys
+sys.path.append(os.environ['ceroot'])
+from kpi import CostKpi, DurationKpi, AccKpi
+
+# NOTE kpi.py should shared in models in some way!!!!
+
+train_cost_kpi = CostKpi('train_cost', 0.02, actived=True)
+test_acc_kpi = AccKpi('test_acc', 0.005, actived=True)
+train_duration_kpi = DurationKpi('train_duration', 0.06, actived=True)
+train_acc_kpi = AccKpi('train_acc', 0.005, actived=True)
+
+tracking_kpis = [
+    train_acc_kpi,
+    train_cost_kpi,
+    test_acc_kpi,
+    train_duration_kpi,
+]
+
+def parse_log(log):
+    '''
+    This method should be implemented by model developers.
+
+    The suggestion:
+
+    each line in the log should be key, value, for example:
+
+    "
+    train_cost\t1.0
+    test_cost\t1.0
+    train_cost\t1.0
+    train_cost\t1.0
+    train_acc\t1.2
+    "
+    '''
+    for line in log.split('\n'):
+        fs = line.strip().split('\t')
+        print (fs)
+        if len(fs) == 3 and fs[0] == 'kpis':
+            kpi_name = fs[1]
+            kpi_value = float(fs[2])
+            yield kpi_name, kpi_value
+
+
+def log_to_ce(log):
+    kpi_tracker = {}
+    for kpi in tracking_kpis:
+        kpi_tracker[kpi.name] = kpi
+
+    for (kpi_name, kpi_value) in parse_log(log):
+        print (kpi_name, kpi_value)
+        kpi_tracker[kpi_name].add_record(kpi_value)
+        kpi_tracker[kpi_name].persist()
+
+
+if __name__ == '__main__':
+    log = sys.stdin.read()
+    log_to_ce(log) 
+

fluid/mnist/model.py

@@ -181,15 +181,12 @@ def run_benchmark(model, args):
             "pass=%d, train_avg_acc=%f,train_avg_loss=%f, test_avg_acc=%f, elapse=%f"
             % (pass_id, train_avg_acc, train_avg_loss, test_avg_acc,
                (pass_end - pass_start)))
-
-        with open("train_acc_factor.txt", 'a+') as f:
-            f.write("%s\n" % train_avg_acc)
-        with open("train_cost_factor.txt", 'a+') as f:
-            f.write('%s\n' % [train_avg_loss])
-        with open("test_acc_factor.txt", 'a+') as f:
-            f.write("%s\n" % test_avg_acc)
-        with open("train_duration_factor.txt", 'a+') as f:
-            f.write('%s\n' % [pass_end - pass_start])
+        #Note: The following logs are special for CE monitoring.
+        #Other situations do not need to care about these logs.
+        print ("kpis	train_acc	%f" % train_avg_acc)
+        print ("kpis	train_cost	%f" % train_avg_loss)  
+        print ("kpis	test_acc	%f" % test_avg_acc)  
+        print ("kpis	train_duration	%f" % (pass_end - pass_start))  
 
 
 if __name__ == '__main__':