simplified captured out implementation hopefully also fixing the prob…

…lems with windows

sacred/observers/base.py

@@ -18,7 +18,7 @@ def started_event(self, ex_info, command, host_info, start_time, config,
                       meta_info, _id):
         pass
 
-    def heartbeat_event(self, info, cout_filename, beat_time):
+    def heartbeat_event(self, info, captured_out, beat_time):
         pass
 
     def completed_event(self, stop_time, result):

sacred/observers/file_storage.py

@@ -157,10 +157,11 @@ def render_template(self):
             with open(os.path.join(self.dir, 'report' + ext), 'w') as f:
                 f.write(report)
 
-    def heartbeat_event(self, info, cout_filename, beat_time):
+    def heartbeat_event(self, info, captured_out, beat_time):
         self.info = info
         self.run_entry['heartbeat'] = beat_time
-        self.save_file(cout_filename, 'cout.txt')
+        self.cout = captured_out
+        self.save_cout()
         self.save_json(self.run_entry, 'run.json')
         if self.info:
             self.save_json(self.info, 'info.json')

sacred/observers/mongo.py

@@ -164,10 +164,9 @@ def started_event(self, ex_info, command, host_info, start_time, config,
         self.save()
         return self.run_entry['_id']
 
-    def heartbeat_event(self, info, cout_filename, beat_time):
+    def heartbeat_event(self, info, captured_out, beat_time):
         self.run_entry['info'] = info
-        with open(cout_filename, 'r') as f:
-            self.run_entry['captured_out'] = f.read()
+        self.run_entry['captured_out'] = captured_out
         self.run_entry['heartbeat'] = beat_time
         self.save()
 

sacred/observers/sql.py

@@ -321,10 +321,9 @@ def queued_event(self, ex_info, command, queue_time, config, meta_info,
         self.session.commit()
         return _id or self.run.run_id
 
-    def heartbeat_event(self, info, cout_filename, beat_time):
+    def heartbeat_event(self, info, captured_out, beat_time):
         self.run.info = json.encode(info)
-        with open(cout_filename, 'r') as f:
-            self.run.captured_out = f.read()
+        self.run.captured_out = captured_out
         self.run.heartbeat = beat_time
         self.session.commit()
 

sacred/observers/tinydb_hashfs.py

@@ -219,10 +219,9 @@ def started_event(self, ex_info, command, host_info, start_time, config,
         self.save()
         return self.run_entry['_id']
 
-    def heartbeat_event(self, info, cout_filename, beat_time):
+    def heartbeat_event(self, info, captured_out, beat_time):
         self.run_entry['info'] = info
-        with open(cout_filename, 'r') as f:
-            self.run_entry['captured_out'] = f.read()
+        self.run_entry['captured_out'] = captured_out
         self.run_entry['heartbeat'] = beat_time
         self.save()
 

sacred/run.py

@@ -170,36 +170,30 @@ def __call__(self, *args):
         if self.queue_only:
             self._emit_queued()
             return
-        try:
-            with tempfile.NamedTemporaryFile(delete=False) as f, tee_output(f):
-                self._output_file = f.name
-                self._emit_started()
-                self._start_heartbeat()
-                try:
-                    self._execute_pre_run_hooks()
-                    self.result = self.main_function(*args)
-                    self._execute_post_run_hooks()
-                    self._stop_heartbeat()
-                    self._emit_completed(self.result)
-                except (SacredInterrupt, KeyboardInterrupt) as e:
-                    self._stop_heartbeat()
-                    status = getattr(e, 'STATUS', 'INTERRUPTED')
-                    self._emit_interrupted(status)
-                    raise
-                except:
-                    exc_type, exc_value, trace = sys.exc_info()
-                    self._stop_heartbeat()
-                    self._emit_failed(exc_type, exc_value, trace.tb_next)
-                    raise
-                finally:
-                    self._warn_about_failed_observers()
-
-            with open(self._output_file, 'r') as f:
-                self.captured_out = f.read()
-        finally:
-            if self._output_file:
-                os.remove(self._output_file)
-
+        with tempfile.NamedTemporaryFile() as f, tee_output(f):
+            self._output_file = f
+            self._emit_started()
+            self._start_heartbeat()
+            try:
+                self._execute_pre_run_hooks()
+                self.result = self.main_function(*args)
+                self._execute_post_run_hooks()
+                self._stop_heartbeat()
+                self._emit_completed(self.result)
+            except (SacredInterrupt, KeyboardInterrupt) as e:
+                self._stop_heartbeat()
+                status = getattr(e, 'STATUS', 'INTERRUPTED')
+                self._emit_interrupted(status)
+                raise
+            except:
+                exc_type, exc_value, trace = sys.exc_info()
+                self._stop_heartbeat()
+                self._emit_failed(exc_type, exc_value, trace.tb_next)
+                raise
+            finally:
+                self._warn_about_failed_observers()
+                f.seek(0)
+                self.captured_out = f.read().decode()
         return self.result
 
     def _start_heartbeat(self):
@@ -267,10 +261,12 @@ def _emit_started(self):
 
     def _emit_heartbeat(self):
         beat_time = datetime.datetime.now()
+        self._output_file.seek(0)
+        captured_out = self._output_file.read().decode()
         for observer in self.observers:
             self._safe_call(observer, 'heartbeat_event',
                             info=self.info,
-                            cout_filename=self._output_file,
+                            captured_out=captured_out,
                             beat_time=beat_time)
 
     def _stop_time(self):

tests/test_observers/test_file_storage_observer.py

@@ -98,10 +98,7 @@ def test_fs_observer_heartbeat_event_updates_run(dir_obs, sample_run):
     _id = obs.started_event(**sample_run)
     run_dir = basedir.join(_id)
     info = {'my_info': [1, 2, 3], 'nr': 7}
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(b'some output')
-        f.flush()
-        obs.heartbeat_event(info=info, cout_filename=f.name, beat_time=T2)
+    obs.heartbeat_event(info=info, captured_out='some output', beat_time=T2)
 
     assert run_dir.join('cout.txt').read() == 'some output'
     run = json.decode(run_dir.join('run.json').read())

tests/test_observers/test_mongo_observer.py

@@ -93,11 +93,7 @@ def test_mongo_observer_heartbeat_event_updates_run(mongo_obs, sample_run):
 
     info = {'my_info': [1, 2, 3], 'nr': 7}
     outp = 'some output'
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(outp.encode())
-        f.flush()
-        mongo_obs.heartbeat_event(info=info, cout_filename=f.name,
-                                  beat_time=T2)
+    mongo_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
 
     assert mongo_obs.runs.count() == 1
     db_run = mongo_obs.runs.find_one()

tests/test_observers/test_sql_observer.py

@@ -148,11 +148,7 @@ def test_sql_observer_heartbeat_event_updates_run(sql_obs, sample_run, session):
 
     info = {'my_info': [1, 2, 3], 'nr': 7}
     outp = 'some output'
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(outp.encode())
-        f.flush()
-        sql_obs.heartbeat_event(info=info, cout_filename=f.name,
-                                beat_time=T2)
+    sql_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
 
     assert session.query(Run).count() == 1
     db_run = session.query(Run).first()

tests/test_observers/test_tinydb_observer.py

@@ -179,11 +179,7 @@ def test_tinydb_observer_heartbeat_event_updates_run(tinydb_obs, sample_run):
 
     info = {'my_info': [1, 2, 3], 'nr': 7}
     outp = 'some output'
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(outp.encode())
-        f.flush()
-        tinydb_obs.heartbeat_event(info=info, cout_filename=f.name,
-                                   beat_time=T2)
+    tinydb_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
 
     assert len(tinydb_obs.runs) == 1
     db_run = tinydb_obs.runs.get(eid=1)

tests/test_observers/test_tinydb_reader.py

@@ -70,11 +70,7 @@ def run_test_experiment(exp_name, exp_id, root_dir):
     # Heartbeat
     info = {'my_info': [1, 2, 3], 'nr': 7}
     outp = 'some output'
-    with tempfile.NamedTemporaryFile() as f:
-        f.write(outp.encode())
-        f.flush()
-        tinydb_obs.heartbeat_event(info=info, cout_filename=f.name,
-                                   beat_time=T2)
+    tinydb_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
     # Add Artifact
     filename = "sacred/__about__.py"
     name = 'about'

tests/test_run.py

@@ -120,7 +120,7 @@ def test_run_heartbeat_event(run):
     run()
     call_args, call_kwargs = observer.heartbeat_event.call_args_list[0]
     assert call_kwargs['info'] == run.info
-    assert call_kwargs['cout_filename'] == run._output_file
+    assert call_kwargs['captured_out'] == ""
     assert (call_kwargs['beat_time'] - datetime.now()).total_seconds() < 1