From bf33664a47dbe8831358195cb9c951be6c26baf2 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Sat, 15 May 2021 14:09:24 +0200 Subject: [PATCH 1/3] log error json serialization --- .../camera_support/prepare_blackvue_videos.py | 2 ++ mapillary_tools/processing.py | 36 ++++++++++--------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/mapillary_tools/camera_support/prepare_blackvue_videos.py b/mapillary_tools/camera_support/prepare_blackvue_videos.py index 2b49c4a65..79fcfa254 100644 --- a/mapillary_tools/camera_support/prepare_blackvue_videos.py +++ b/mapillary_tools/camera_support/prepare_blackvue_videos.py @@ -6,6 +6,8 @@ def find_camera_model(videos_folder): from mapillary_tools.uploader import get_video_file_list file_list = get_video_file_list(videos_folder) + if not file_list: + raise RuntimeError(f"No video found in {videos_folder}") fd = open(file_list[0], "rb") diff --git a/mapillary_tools/processing.py b/mapillary_tools/processing.py index d45fb7dba..45c72da36 100644 --- a/mapillary_tools/processing.py +++ b/mapillary_tools/processing.py @@ -229,7 +229,7 @@ def geotag_from_gopro_video( f"Error, no video frames extracted for video file {gopro_video} in import_path {import_path}" ) create_and_log_process_in_list( - process_file_sublist, "geotag_process" "failed", verbose + process_file_sublist, "geotag_process", "failed", verbose=verbose ) continue @@ -268,15 +268,18 @@ def geotag_from_blackvue_video( [gpx_path, is_stationary_video] = gpx_from_blackvue( blackvue_video, use_nmea_stream_timestamp=False ) - if not gpx_path or not os.path.isfile(gpx_path): - raise Exception except Exception as e: - print_error( + raise RuntimeError( f"Error, failed extracting data from blackvue geotag source path {blackvue_video} due to {e}, exiting..." ) + + if not gpx_path or not os.path.isfile(gpx_path): + raise RuntimeError("Error, GPX path {gpx_path} not found") + if is_stationary_video: print_error("Warning: Skipping stationary video") continue + process_file_sublist = [ x for x in process_file_list @@ -288,7 +291,7 @@ def geotag_from_blackvue_video( f"Error, no video frames extracted for video file {blackvue_video} in import_path {import_path}" ) create_and_log_process_in_list( - process_file_sublist, "geotag_process" "failed", verbose + process_file_sublist, "geotag_process", "failed", verbose=verbose ) continue @@ -333,6 +336,8 @@ def geotag_from_gps_trace( gps_trace = get_lat_lon_time_from_gpx(geotag_source_path, local_time) elif geotag_source == "nmea": gps_trace = get_lat_lon_time_from_nmea(geotag_source_path, local_time) + else: + raise RuntimeError(f"Invalid geotag source {geotag_source}") # Estimate capture time with sub-second precision, reading from image EXIF sub_second_times = estimate_sub_second_time(process_file_list, sub_second_interval) @@ -341,7 +346,7 @@ def geotag_from_gps_trace( "Error, capture times could not be estimated to sub second precision, images can not be geotagged." ) create_and_log_process_in_list( - process_file_list, "geotag_process" "failed", verbose + process_file_list, "geotag_process", "failed", verbose=verbose ) return @@ -350,7 +355,7 @@ def geotag_from_gps_trace( f"Error, gps trace file {geotag_source_path} was not read, images can not be geotagged." ) create_and_log_process_in_list( - process_file_list, "geotag_process", "failed", verbose + process_file_list, "geotag_process", "failed", verbose=verbose ) return @@ -449,22 +454,17 @@ def get_upload_param_properties( # load the sequence json user_process_json_path = os.path.join(log_root, "user_process.json") - user_data = {} try: user_data = load_json(user_process_json_path) except: print( - "Warning, user data not read for image " - + image - + ", therefore it will not be included in the upload params processing." + f"Warning, user data not read for image {image}, therefore it will not be included in the upload params processing." ) return None if "MAPSettingsUserKey" not in user_data: print( - "Warning, user key not in user data for image " - + image - + ", therefore it will not be included in the upload params processing." + "Warning, user key not in user data for image {image}, therefore it will not be included in the upload params processing." ) return None @@ -749,8 +749,12 @@ def load_json(file_path: str): def save_json(data: Dict[str, Any], file_path: str) -> None: - with open(file_path, "wb") as f: - f.write(json.dumps(data, indent=4).encode("utf-8")) + try: + buf = json.dumps(data, indent=4) + except Exception: + raise RuntimeError(f"Error JSON serializing {data}") + with open(file_path, "w") as f: + f.write(buf) def update_json(data, file_path, process): From 88d93874b40df9e2d9cd45532885f50dcb36e268 Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Tue, 18 May 2021 20:46:52 +0200 Subject: [PATCH 2/3] fix: convert devide model to unicode --- mapillary_tools/camera_support/prepare_blackvue_videos.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mapillary_tools/camera_support/prepare_blackvue_videos.py b/mapillary_tools/camera_support/prepare_blackvue_videos.py index 79fcfa254..2a1f4f529 100644 --- a/mapillary_tools/camera_support/prepare_blackvue_videos.py +++ b/mapillary_tools/camera_support/prepare_blackvue_videos.py @@ -2,7 +2,7 @@ import io -def find_camera_model(videos_folder): +def find_camera_model(videos_folder) -> bytes: from mapillary_tools.uploader import get_video_file_list file_list = get_video_file_list(videos_folder) @@ -18,10 +18,12 @@ def find_camera_model(videos_folder): box = Box.parse_stream(fd) if box.type.decode("utf-8") == "free": # or 'ftyp': return box.data[29:39] + raise RuntimeError(f"camera model not found in {file_list[0]}") def apply_config_blackvue(vars_args): - vars_args["device_model"] = find_camera_model(vars_args["video_import_path"]) + model = find_camera_model(vars_args["video_import_path"]) + vars_args["device_model"] = model.decode("utf-8") vars_args["device_make"] = "Blackvue" vars_args["geotag_source"] = "blackvue_videos" vars_args["duplicate_angle"] = 360 From e8dc2f31ea75ac41e4c486c6bbb53cb4955af99b Mon Sep 17 00:00:00 2001 From: Tao Peng Date: Thu, 3 Jun 2021 14:57:55 +0200 Subject: [PATCH 3/3] convert bytes to str after pymp4 parsing --- mapillary_tools/gpx_from_blackvue.py | 42 ++++++++++------------------ mapillary_tools/processing.py | 13 +++------ 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/mapillary_tools/gpx_from_blackvue.py b/mapillary_tools/gpx_from_blackvue.py index 9f3497447..1643bd79a 100644 --- a/mapillary_tools/gpx_from_blackvue.py +++ b/mapillary_tools/gpx_from_blackvue.py @@ -1,3 +1,4 @@ +from typing import Tuple import datetime import io import os @@ -48,7 +49,8 @@ def get_points_from_bv(path, use_nmea_stream_timestamp=False): lines = newb.data # Parse GPS trace - for line in lines.splitlines(): + for line_bytes in lines.splitlines(): + line = line_bytes.decode("utf-8") m = line.lstrip("[]0123456789") # this utc millisecond timestamp seems to be the camera's # todo: unused? @@ -180,37 +182,21 @@ def get_points_from_bv(path, use_nmea_stream_timestamp=False): return points -def is_video_stationary(max_distance_from_start): - radius_treshold = 10 +def is_video_stationary(max_distance_from_start) -> bool: + radius_threshold = 10 accumulated_distance_threshold = 20 - - if ( - max_distance_from_start < radius_treshold + return ( + max_distance_from_start < radius_threshold or accumulated_distance_threshold < accumulated_distance_threshold - ): - return True - else: - return False - - -def gpx_from_blackvue(bv_video, use_nmea_stream_timestamp=False): - bv_data = [] - try: - bv_data = get_points_from_bv(bv_video, use_nmea_stream_timestamp) - except Exception as e: - print( - f"Warning, could not extract gps from video {bv_video} due to {e}, video will be skipped..." - ) + ) + + +def gpx_from_blackvue(bv_video, use_nmea_stream_timestamp=False) -> Tuple[str, bool]: + bv_data = get_points_from_bv(bv_video, use_nmea_stream_timestamp) if not bv_data: - is_stationary_video = is_video_stationary(get_max_distance_from_start(bv_data)) - else: - is_stationary_video = True - return [], is_stationary_video + return "", True basename, extension = os.path.splitext(bv_video) gpx_path = basename + ".gpx" - bv_data.sort(key=lambda x: x[0]) - write_gpx(gpx_path, bv_data) - - return gpx_path, is_stationary_video + return gpx_path, is_video_stationary(get_max_distance_from_start(bv_data)) diff --git a/mapillary_tools/processing.py b/mapillary_tools/processing.py index 45c72da36..a058f040c 100644 --- a/mapillary_tools/processing.py +++ b/mapillary_tools/processing.py @@ -264,17 +264,12 @@ def geotag_from_blackvue_video( blackvue_video_filename = ( os.path.basename(blackvue_video).replace(".mp4", "").replace(".MP4", "") ) - try: - [gpx_path, is_stationary_video] = gpx_from_blackvue( - blackvue_video, use_nmea_stream_timestamp=False - ) - except Exception as e: - raise RuntimeError( - f"Error, failed extracting data from blackvue geotag source path {blackvue_video} due to {e}, exiting..." - ) + [gpx_path, is_stationary_video] = gpx_from_blackvue( + blackvue_video, use_nmea_stream_timestamp=False + ) if not gpx_path or not os.path.isfile(gpx_path): - raise RuntimeError("Error, GPX path {gpx_path} not found") + raise RuntimeError(f"Error, GPX path {gpx_path} not found") if is_stationary_video: print_error("Warning: Skipping stationary video")