Fix JSON serialization in blackvue processing

mapillary_tools/camera_support/prepare_blackvue_videos.py

@@ -2,10 +2,12 @@
 import io
 
 
-def find_camera_model(videos_folder):
+def find_camera_model(videos_folder) -> bytes:
     from mapillary_tools.uploader import get_video_file_list
 
     file_list = get_video_file_list(videos_folder)
+    if not file_list:
+        raise RuntimeError(f"No video found in {videos_folder}")
 
     fd = open(file_list[0], "rb")
 
@@ -16,10 +18,12 @@ def find_camera_model(videos_folder):
         box = Box.parse_stream(fd)
         if box.type.decode("utf-8") == "free":  # or 'ftyp':
             return box.data[29:39]
+    raise RuntimeError(f"camera model not found in {file_list[0]}")
 
 
 def apply_config_blackvue(vars_args):
-    vars_args["device_model"] = find_camera_model(vars_args["video_import_path"])
+    model = find_camera_model(vars_args["video_import_path"])
+    vars_args["device_model"] = model.decode("utf-8")
     vars_args["device_make"] = "Blackvue"
     vars_args["geotag_source"] = "blackvue_videos"
     vars_args["duplicate_angle"] = 360

mapillary_tools/gpx_from_blackvue.py

@@ -1,3 +1,4 @@
+from typing import Tuple
 import datetime
 import io
 import os
@@ -48,7 +49,8 @@ def get_points_from_bv(path, use_nmea_stream_timestamp=False):
                         lines = newb.data
 
                         # Parse GPS trace
-                        for line in lines.splitlines():
+                        for line_bytes in lines.splitlines():
+                            line = line_bytes.decode("utf-8")
                             m = line.lstrip("[]0123456789")
                             # this utc millisecond timestamp seems to be the camera's
                             # todo: unused?
@@ -180,37 +182,21 @@ def get_points_from_bv(path, use_nmea_stream_timestamp=False):
         return points
 
 
-def is_video_stationary(max_distance_from_start):
-    radius_treshold = 10
+def is_video_stationary(max_distance_from_start) -> bool:
+    radius_threshold = 10
     accumulated_distance_threshold = 20
-
-    if (
-        max_distance_from_start < radius_treshold
+    return (
+        max_distance_from_start < radius_threshold
         or accumulated_distance_threshold < accumulated_distance_threshold
-    ):
-        return True
-    else:
-        return False
-
-
-def gpx_from_blackvue(bv_video, use_nmea_stream_timestamp=False):
-    bv_data = []
-    try:
-        bv_data = get_points_from_bv(bv_video, use_nmea_stream_timestamp)
-    except Exception as e:
-        print(
-            f"Warning, could not extract gps from video {bv_video} due to {e}, video will be skipped..."
-        )
+    )
+
+
+def gpx_from_blackvue(bv_video, use_nmea_stream_timestamp=False) -> Tuple[str, bool]:
+    bv_data = get_points_from_bv(bv_video, use_nmea_stream_timestamp)
     if not bv_data:
-        is_stationary_video = is_video_stationary(get_max_distance_from_start(bv_data))
-    else:
-        is_stationary_video = True
-        return [], is_stationary_video
+        return "", True
     basename, extension = os.path.splitext(bv_video)
     gpx_path = basename + ".gpx"
-
     bv_data.sort(key=lambda x: x[0])
-
     write_gpx(gpx_path, bv_data)
-
-    return gpx_path, is_stationary_video
+    return gpx_path, is_video_stationary(get_max_distance_from_start(bv_data))

mapillary_tools/processing.py

@@ -229,7 +229,7 @@ def geotag_from_gopro_video(
                 f"Error, no video frames extracted for video file {gopro_video} in import_path {import_path}"
             )
             create_and_log_process_in_list(
-                process_file_sublist, "geotag_process" "failed", verbose
+                process_file_sublist, "geotag_process", "failed", verbose=verbose
             )
             continue
 
@@ -264,19 +264,17 @@ def geotag_from_blackvue_video(
         blackvue_video_filename = (
             os.path.basename(blackvue_video).replace(".mp4", "").replace(".MP4", "")
         )
-        try:
-            [gpx_path, is_stationary_video] = gpx_from_blackvue(
-                blackvue_video, use_nmea_stream_timestamp=False
-            )
-            if not gpx_path or not os.path.isfile(gpx_path):
-                raise Exception
-        except Exception as e:
-            print_error(
-                f"Error, failed extracting data from blackvue geotag source path {blackvue_video} due to {e}, exiting..."
-            )
+        [gpx_path, is_stationary_video] = gpx_from_blackvue(
+            blackvue_video, use_nmea_stream_timestamp=False
+        )
+
+        if not gpx_path or not os.path.isfile(gpx_path):
+            raise RuntimeError(f"Error, GPX path {gpx_path} not found")
+
         if is_stationary_video:
             print_error("Warning: Skipping stationary video")
             continue
+
         process_file_sublist = [
             x
             for x in process_file_list
@@ -288,7 +286,7 @@ def geotag_from_blackvue_video(
                 f"Error, no video frames extracted for video file {blackvue_video} in import_path {import_path}"
             )
             create_and_log_process_in_list(
-                process_file_sublist, "geotag_process" "failed", verbose
+                process_file_sublist, "geotag_process", "failed", verbose=verbose
             )
             continue
 
@@ -333,6 +331,8 @@ def geotag_from_gps_trace(
         gps_trace = get_lat_lon_time_from_gpx(geotag_source_path, local_time)
     elif geotag_source == "nmea":
         gps_trace = get_lat_lon_time_from_nmea(geotag_source_path, local_time)
+    else:
+        raise RuntimeError(f"Invalid geotag source {geotag_source}")
 
     # Estimate capture time with sub-second precision, reading from image EXIF
     sub_second_times = estimate_sub_second_time(process_file_list, sub_second_interval)
@@ -341,7 +341,7 @@ def geotag_from_gps_trace(
             "Error, capture times could not be estimated to sub second precision, images can not be geotagged."
         )
         create_and_log_process_in_list(
-            process_file_list, "geotag_process" "failed", verbose
+            process_file_list, "geotag_process", "failed", verbose=verbose
         )
         return
 
@@ -350,7 +350,7 @@ def geotag_from_gps_trace(
             f"Error, gps trace file {geotag_source_path} was not read, images can not be geotagged."
         )
         create_and_log_process_in_list(
-            process_file_list, "geotag_process", "failed", verbose
+            process_file_list, "geotag_process", "failed", verbose=verbose
         )
         return
 
@@ -449,22 +449,17 @@ def get_upload_param_properties(
 
     # load the sequence json
     user_process_json_path = os.path.join(log_root, "user_process.json")
-    user_data = {}
     try:
         user_data = load_json(user_process_json_path)
     except:
         print(
-            "Warning, user data not read for image "
-            + image
-            + ", therefore it will not be included in the upload params processing."
+            f"Warning, user data not read for image {image}, therefore it will not be included in the upload params processing."
         )
         return None
 
     if "MAPSettingsUserKey" not in user_data:
         print(
-            "Warning, user key not in user data for image "
-            + image
-            + ", therefore it will not be included in the upload params processing."
+            "Warning, user key not in user data for image {image}, therefore it will not be included in the upload params processing."
         )
         return None
 
@@ -749,8 +744,12 @@ def load_json(file_path: str):
 
 
 def save_json(data: Dict[str, Any], file_path: str) -> None:
-    with open(file_path, "wb") as f:
-        f.write(json.dumps(data, indent=4).encode("utf-8"))
+    try:
+        buf = json.dumps(data, indent=4)
+    except Exception:
+        raise RuntimeError(f"Error JSON serializing {data}")
+    with open(file_path, "w") as f:
+        f.write(buf)
 
 
 def update_json(data, file_path, process):