From 5c382c3568f7498286e0709d37f26bdef2d576df Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Tue, 11 Dec 2018 16:52:40 +1000 Subject: [PATCH 1/9] Always Compress - Make sure uploader to bzip2 rlog once it's completed without any upload permission. --- selfdrive/loggerd/uploader.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 2d793064af83d0..6de8953fb4649e 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -197,7 +197,7 @@ def abort_upload(self): raise_on_thread(thread, SystemExit) thread.join() - def upload(self, key, fn): + def compress(self, key, fn): # write out the bz2 compress if fn.endswith("log"): ext = ".bz2" @@ -210,6 +210,9 @@ def upload(self, key, fn): key += ext fn += ext + return key, fn + + def upload(self, key, fn): try: sz = os.path.getsize(fn) except OSError: @@ -262,17 +265,19 @@ def uploader_fn(exit_event): if exit_event.is_set(): return - if not should_upload: + d = uploader.next_file_to_upload(with_video=True) + key, fn, _ = d + + if d is not None: + key, fn = uploader.compress(key, fn) + else: time.sleep(5) continue - d = uploader.next_file_to_upload(with_video=True) - if d is None: + if not should_upload: time.sleep(5) continue - key, fn, _ = d - cloudlog.info("to upload %r", d) success = uploader.upload(key, fn) if success: From 976df732c3ca8a16dd95e87c3cecb9f222f60889 Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Fri, 14 Dec 2018 10:18:30 +1000 Subject: [PATCH 2/9] fix up logic. --- selfdrive/loggerd/uploader.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 6de8953fb4649e..62e22a735da8f6 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -123,10 +123,15 @@ def get_data_stats(self): total_size += os.stat(fn).st_size return dict(name_counts), total_size + def next_file_to_compress(self): + for name, key, fn in self.gen_upload_files(): + if name in ["rlog"]: + return (key, fn, 0) + def next_file_to_upload(self, with_video): # try to upload log files first for name, key, fn in self.gen_upload_files(): - if name in ["rlog", "rlog.bz2"]: + if name == "rlog.bz2": return (key, fn, 0) if with_video: @@ -265,16 +270,20 @@ def uploader_fn(exit_event): if exit_event.is_set(): return - d = uploader.next_file_to_upload(with_video=True) + d = uploader.next_file_to_compress() key, fn, _ = d if d is not None: - key, fn = uploader.compress(key, fn) - else: + uploader.compress(key, fn) + + if not should_upload: time.sleep(5) continue - if not should_upload: + d = uploader.next_file_to_upload(with_video=True) + key, fn, _ = d + + if d is None: time.sleep(5) continue From 2f231b06b9dd3062515f6d27dfe04d893972675e Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Fri, 14 Dec 2018 16:18:06 +1000 Subject: [PATCH 3/9] fix up logic --- selfdrive/loggerd/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 62e22a735da8f6..d682ec15929217 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -125,8 +125,9 @@ def get_data_stats(self): def next_file_to_compress(self): for name, key, fn in self.gen_upload_files(): - if name in ["rlog"]: + if name == "rlog": return (key, fn, 0) + return None def next_file_to_upload(self, with_video): # try to upload log files first From fce340cc2e784bb6bc3a7c103361dc99d22c4254 Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Fri, 14 Dec 2018 16:18:06 +1000 Subject: [PATCH 4/9] fix up logic --- selfdrive/loggerd/uploader.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 62e22a735da8f6..58e9640a14307a 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -125,8 +125,9 @@ def get_data_stats(self): def next_file_to_compress(self): for name, key, fn in self.gen_upload_files(): - if name in ["rlog"]: + if name == "rlog": return (key, fn, 0) + return None def next_file_to_upload(self, with_video): # try to upload log files first @@ -271,9 +272,9 @@ def uploader_fn(exit_event): return d = uploader.next_file_to_compress() - key, fn, _ = d if d is not None: + key, fn, _ = d uploader.compress(key, fn) if not should_upload: @@ -281,12 +282,13 @@ def uploader_fn(exit_event): continue d = uploader.next_file_to_upload(with_video=True) - key, fn, _ = d if d is None: time.sleep(5) continue + key, fn, _ = d + cloudlog.info("to upload %r", d) success = uploader.upload(key, fn) if success: From 6f58df32326e88d1dfaeff9c4d70fee6daefe710 Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Tue, 18 Dec 2018 13:23:04 +1000 Subject: [PATCH 5/9] minimise changes. --- selfdrive/loggerd/uploader.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 58e9640a14307a..4338f8f3eded72 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -123,16 +123,10 @@ def get_data_stats(self): total_size += os.stat(fn).st_size return dict(name_counts), total_size - def next_file_to_compress(self): - for name, key, fn in self.gen_upload_files(): - if name == "rlog": - return (key, fn, 0) - return None - def next_file_to_upload(self, with_video): # try to upload log files first for name, key, fn in self.gen_upload_files(): - if name == "rlog.bz2": + if name in ["rlog", "rlog.bz2"]: return (key, fn, 0) if with_video: @@ -216,7 +210,7 @@ def compress(self, key, fn): key += ext fn += ext - return key, fn + return (key, fn) def upload(self, key, fn): try: @@ -271,16 +265,6 @@ def uploader_fn(exit_event): if exit_event.is_set(): return - d = uploader.next_file_to_compress() - - if d is not None: - key, fn, _ = d - uploader.compress(key, fn) - - if not should_upload: - time.sleep(5) - continue - d = uploader.next_file_to_upload(with_video=True) if d is None: @@ -289,6 +273,13 @@ def uploader_fn(exit_event): key, fn, _ = d + if _ == 0: + key, fn = self.compress(key, fn) + + if not should_upload: + time.sleep(5) + continue + cloudlog.info("to upload %r", d) success = uploader.upload(key, fn) if success: From bd3a2d93ce8d5ab0989909e36a6d3c589044b1d0 Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Wed, 19 Dec 2018 10:18:22 +1000 Subject: [PATCH 6/9] fix error. --- selfdrive/loggerd/uploader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 4338f8f3eded72..ff1e4e37663442 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -272,9 +272,8 @@ def uploader_fn(exit_event): continue key, fn, _ = d - if _ == 0: - key, fn = self.compress(key, fn) + key, fn = uploader.compress(key, fn) if not should_upload: time.sleep(5) From 98d7c452a427d47b5b8c06f9e9468bd796eb3220 Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Thu, 20 Dec 2018 09:02:34 +1000 Subject: [PATCH 7/9] let uploader to upload compressed log first (when allowed) then process uncompressed rlog (suggested by ku7) --- selfdrive/loggerd/uploader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index ff1e4e37663442..4a0698285d576e 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -124,9 +124,14 @@ def get_data_stats(self): return dict(name_counts), total_size def next_file_to_upload(self, with_video): - # try to upload log files first + # try to upload compressed log files first for name, key, fn in self.gen_upload_files(): - if name in ["rlog", "rlog.bz2"]: + if name == "rlog.bz2": + return (key, fn, 0) + + # then upload those which are yet to be compressed + for name, key, fn in self.gen_upload_files(): + if name == "rlog": return (key, fn, 0) if with_video: From 55bb253734aa9deda52067bbcd35cacdaaf200bd Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Fri, 21 Dec 2018 15:18:32 +1000 Subject: [PATCH 8/9] Revert "let uploader to upload compressed log first (when allowed) then process uncompressed rlog (suggested by ku7)" This reverts commit 98d7c452a427d47b5b8c06f9e9468bd796eb3220. --- selfdrive/loggerd/uploader.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index 4a0698285d576e..ff1e4e37663442 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -124,14 +124,9 @@ def get_data_stats(self): return dict(name_counts), total_size def next_file_to_upload(self, with_video): - # try to upload compressed log files first + # try to upload log files first for name, key, fn in self.gen_upload_files(): - if name == "rlog.bz2": - return (key, fn, 0) - - # then upload those which are yet to be compressed - for name, key, fn in self.gen_upload_files(): - if name == "rlog": + if name in ["rlog", "rlog.bz2"]: return (key, fn, 0) if with_video: From 2afbc1205af69f28ec098ceecadba8d3f310931f Mon Sep 17 00:00:00 2001 From: Rick Lan Date: Fri, 21 Dec 2018 18:15:24 +1000 Subject: [PATCH 9/9] Update logic, make sure it compress first regardless. --- selfdrive/loggerd/uploader.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/selfdrive/loggerd/uploader.py b/selfdrive/loggerd/uploader.py index ff1e4e37663442..47be81f127f518 100644 --- a/selfdrive/loggerd/uploader.py +++ b/selfdrive/loggerd/uploader.py @@ -123,10 +123,16 @@ def get_data_stats(self): total_size += os.stat(fn).st_size return dict(name_counts), total_size + def next_file_to_compress(self): + for name, key, fn in self.gen_upload_files(): + if name == "rlog": + return (key, fn, 0) + return None + def next_file_to_upload(self, with_video): # try to upload log files first for name, key, fn in self.gen_upload_files(): - if name in ["rlog", "rlog.bz2"]: + if name == "rlog.bz2": return (key, fn, 0) if with_video: @@ -265,20 +271,23 @@ def uploader_fn(exit_event): if exit_event.is_set(): return - d = uploader.next_file_to_upload(with_video=True) + d = uploader.next_file_to_compress() + if d is not None: + key, fn, _ = d + uploader.compress(key, fn) + continue - if d is None: + if not should_upload: time.sleep(5) continue - key, fn, _ = d - if _ == 0: - key, fn = uploader.compress(key, fn) - - if not should_upload: + d = uploader.next_file_to_upload(with_video=True) + if d is None: time.sleep(5) continue + key, fn, _ = d + cloudlog.info("to upload %r", d) success = uploader.upload(key, fn) if success: