Compress when not uploading (commaai#471)

README.md

@@ -37,6 +37,7 @@ Based on OpenPilot 0.5.7 from comma.ai
 - ** Temporarily removed ** CLI Based Real Time Tuning (Thanks @JamesT)
 - Unsupported cars default to Kia Sorento.  This should work for most cars.
 - Cruise Setpoint set from OSM Speed Limit*
+- Compress when not uploading (https://github.com/commaai/openpilot/pull/471)
 - Probably other things I have forgotten
 
 * Has known issues

selfdrive/loggerd/uploader.py

@@ -124,9 +124,14 @@ def get_data_stats(self):
     return dict(name_counts), total_size
 
   def next_file_to_upload(self, with_video):
-    # try to upload log files first
+    # try to upload compressed log files first
     for name, key, fn in self.gen_upload_files():
-      if name in ["rlog", "rlog.bz2"]:
+      if name == "rlog.bz2":
+        return (key, fn, 0)
+
+    # then do those which are yet to be compressed
+    for name, key, fn in self.gen_upload_files():
+      if name == "rlog":
         return (key, fn, 0)
 
     if with_video:
@@ -197,7 +202,7 @@ def abort_upload(self):
     raise_on_thread(thread, SystemExit)
     thread.join()
 
-  def upload(self, key, fn):
+  def compress(self, key, fn):
     # write out the bz2 compress
     if fn.endswith("log"):
       ext = ".bz2"
@@ -209,6 +214,10 @@ def upload(self, key, fn):
       # assuming file is named properly
       key += ext
       fn += ext
+
+    return (key, fn)
+
+  def upload(self, key, fn):
 
     try:
       sz = os.path.getsize(fn)
@@ -262,16 +271,19 @@ def uploader_fn(exit_event):
     if exit_event.is_set():
       return
 
-    if not should_upload:
-      time.sleep(5)
-      continue
-
     d = uploader.next_file_to_upload(with_video=True)
+
     if d is None:
       time.sleep(5)
       continue
 
     key, fn, _ = d
+    if _ == 0:
+      key, fn = uploader.compress(key, fn)
+
+    if not should_upload:
+      time.sleep(5)
+      continue
 
     cloudlog.info("to upload %r", d)
     success = uploader.upload(key, fn)