From ec6153ddbd7f7cc3184a8ca0ff89df30031cbd32 Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 28 May 2025 16:52:58 +0000 Subject: [PATCH 1/2] Validate timestamps. --- .../__init__.py | 66 ++++++++++--------- .../tests/test_llama_video_summarization.py | 56 +++++++++++++++- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 49544129..b3f90868 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -120,36 +120,38 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, response: str ) -> Tuple[Union[dict, None], Union[str, None]]: + error_msg = None response_json = None if not response: - error = 'Empty response.' - log.warning(error) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return None, error - - try: - response_json = json.loads(response) - except ValueError as ve: - error = 'Response is not valid JSON.' - log.warning(error) - log.warning(str(ve)) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error - - try: - validate(response_json, schema_json) - except ValidationError as ve: - error = 'Response JSON is not in the desired format.' - log.warning(error) - log.warning(str(ve)) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error + error_msg = 'Empty response.' + + if not error_msg: + try: + response_json = json.loads(response) + except ValueError as ve: + error_msg = f'Response is not valid JSON. {str(ve)}' + + if not error_msg and response_json: + try: + validate(response_json, schema_json) + except ValidationError as ve: + error_msg = f'Response JSON is not in the desired format. {str(ve)}' - return response_json, None + if not error_msg and response_json: + try: + event_timeline = response_json['video_event_timeline'] + for event in event_timeline: + # update values for later use + event["timestamp_start"] = _get_timestamp_value(event["timestamp_start"]) + event["timestamp_end"] = _get_timestamp_value(event["timestamp_end"]) + except ValueError as ve: + error_msg = f'Response JSON is not in the desired format. {str(ve)}' + + log.warning(error_msg) + log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') + attempts['base'] += 1 + return response_json, error_msg def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, @@ -158,8 +160,8 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, error = None for event in event_timeline: - timestamp_start = _get_timestamp_value(event["timestamp_start"]) - timestamp_end = _get_timestamp_value(event["timestamp_end"]) + timestamp_start = event["timestamp_start"] + timestamp_end = event["timestamp_end"] if timestamp_start < 0: error = (f'Timeline event start time of {timestamp_start} < 0.') @@ -185,7 +187,7 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, break if not error: - min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), + min_event_start = min(list(map(lambda d: d.get('timestamp_start'), filter(lambda d: 'timestamp_start' in d, event_timeline)))) if abs(segment_start_time - min_event_start) > threshold: @@ -193,7 +195,7 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, f'abs({segment_start_time} - {min_event_start}) > {threshold}.') if not error: - max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), + max_event_end = max(list(map(lambda d: d.get('timestamp_end'), filter(lambda d: 'timestamp_end' in d, event_timeline)))) if abs(max_event_end - segment_stop_time) > threshold: @@ -263,8 +265,8 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf for event in response_json['video_event_timeline']: # get offset start/stop times in milliseconds - event_start_time = int(_get_timestamp_value(event['timestamp_start']) * 1000) - event_stop_time = int(_get_timestamp_value(event['timestamp_end']) * 1000) + event_start_time = int(event['timestamp_start'] * 1000) + event_stop_time = int(event['timestamp_end'] * 1000) offset_start_frame = int((event_start_time * video_fps) / 1000) offset_stop_frame = int((event_stop_time * video_fps) / 1000) - 1 diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 78b84079..95f61537 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -69,7 +69,7 @@ }, { "timestamp_start": "5.0", - "timestamp_end": "6.8", + "timestamp_end": "6.8s", "description": "The cat looks back at the camera and then walks away." } ] @@ -340,6 +340,59 @@ def test_invalid_json_response(self): self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("not valid JSON", str(cm.exception)) + + def test_schema_check(self): + component = LlamaVideoSummarizationComponent() + + job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 171, + { + "GENERATION_MAX_ATTEMPTS" : "1" + }, + CAT_VIDEO_PROPERTIES, {}) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps( + { + "video_summary": "This is a video of a cat.", + "video_event_timeline": [ + { + "timestamp_start": "0.00", + "bad": "8.04", + "description": "The cat is sitting on the cobblestone street, looking around." + } + ] + })) # don't care about results + + self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) + self.assertIn("'timestamp_end' is a required property", str(cm.exception)) + + + def test_invalid_timestamp(self): + component = LlamaVideoSummarizationComponent() + + job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 171, + { + "GENERATION_MAX_ATTEMPTS" : "1" + }, + CAT_VIDEO_PROPERTIES, {}) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps( + { + "video_summary": "This is a video of a cat.", + "video_event_timeline": [ + { + "timestamp_start": "7:12", + "timestamp_end": "8:04", + "description": "The cat is sitting on the cobblestone street, looking around." + } + ] + })) # don't care about results + + self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) + self.assertIn("could not convert string to float", str(cm.exception)) + + def test_empty_response(self): component = LlamaVideoSummarizationComponent() @@ -355,6 +408,7 @@ def test_empty_response(self): self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Empty response", str(cm.exception)) + def test_timeline_integrity(self): component = LlamaVideoSummarizationComponent() From 14f30adc27421da3157b1e5fe6be0dbe3df8ecfb Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 28 May 2025 17:06:22 +0000 Subject: [PATCH 2/2] Fix bug. --- .../__init__.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index b3f90868..3cceba56 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -120,25 +120,25 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, response: str ) -> Tuple[Union[dict, None], Union[str, None]]: - error_msg = None + error = None response_json = None if not response: - error_msg = 'Empty response.' + error = 'Empty response.' - if not error_msg: + if not error: try: response_json = json.loads(response) except ValueError as ve: - error_msg = f'Response is not valid JSON. {str(ve)}' + error = f'Response is not valid JSON. {str(ve)}' - if not error_msg and response_json: + if not error and response_json: try: validate(response_json, schema_json) except ValidationError as ve: - error_msg = f'Response JSON is not in the desired format. {str(ve)}' + error = f'Response JSON is not in the desired format. {str(ve)}' - if not error_msg and response_json: + if not error and response_json: try: event_timeline = response_json['video_event_timeline'] for event in event_timeline: @@ -146,12 +146,14 @@ def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, event["timestamp_start"] = _get_timestamp_value(event["timestamp_start"]) event["timestamp_end"] = _get_timestamp_value(event["timestamp_end"]) except ValueError as ve: - error_msg = f'Response JSON is not in the desired format. {str(ve)}' + error = f'Response JSON is not in the desired format. {str(ve)}' - log.warning(error_msg) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error_msg + if error: + log.warning(error) + log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') + attempts['base'] += 1 + + return response_json, error def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int,