From befe776d5e899f8545dcf7b574a7d214e1d25ef2 Mon Sep 17 00:00:00 2001
From: emmertex <andrew@emmertex.com>
Date: Wed, 19 Dec 2018 22:52:19 +1100
Subject: [PATCH] Compress when not uploading
 (https://github.com/commaai/openpilot/pull/471)

---
 README.md                     |  1 +
 selfdrive/loggerd/uploader.py | 26 +++++++++++++++++++-------
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 2fdfcfd1c60cf8..ebbf3f712e197a 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,7 @@ Based on OpenPilot 0.5.7 from comma.ai
 - ** Temporarily removed ** CLI Based Real Time Tuning (Thanks @JamesT)
 - Unsupported cars default to Kia Sorento.  This should work for most cars.
 - Cruise Setpoint set from OSM Speed Limit*
+- Compress when not uploading (https://github.com/commaai/openpilot/pull/471)
 - Probably other things I have forgotten
 
 * Has known issues
diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py
index 2d793064af83d0..8d456710d69e7b 100644
--- a/selfdrive/loggerd/uploader.py
+++ b/selfdrive/loggerd/uploader.py
@@ -124,9 +124,14 @@ def get_data_stats(self):
     return dict(name_counts), total_size
 
   def next_file_to_upload(self, with_video):
-    # try to upload log files first
+    # try to upload compressed log files first
     for name, key, fn in self.gen_upload_files():
-      if name in ["rlog", "rlog.bz2"]:
+      if name == "rlog.bz2":
+        return (key, fn, 0)
+
+    # then do those which are yet to be compressed
+    for name, key, fn in self.gen_upload_files():
+      if name == "rlog":
         return (key, fn, 0)
 
     if with_video:
@@ -197,7 +202,7 @@ def abort_upload(self):
     raise_on_thread(thread, SystemExit)
     thread.join()
 
-  def upload(self, key, fn):
+  def compress(self, key, fn):
     # write out the bz2 compress
     if fn.endswith("log"):
       ext = ".bz2"
@@ -209,6 +214,10 @@ def upload(self, key, fn):
       # assuming file is named properly
       key += ext
       fn += ext
+     
+    return (key, fn)
+
+  def upload(self, key, fn):
 
     try:
       sz = os.path.getsize(fn)
@@ -262,16 +271,19 @@ def uploader_fn(exit_event):
     if exit_event.is_set():
       return
 
-    if not should_upload:
-      time.sleep(5)
-      continue
-
     d = uploader.next_file_to_upload(with_video=True)
+    
     if d is None:
       time.sleep(5)
       continue
 
     key, fn, _ = d
+    if _ == 0:
+      key, fn = uploader.compress(key, fn)
+
+    if not should_upload:
+      time.sleep(5)
+      continue
 
     cloudlog.info("to upload %r", d)
     success = uploader.upload(key, fn)