From 1b535ec1f73a06116f91506ca16d98f0fc12e55b Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Thu, 22 May 2025 15:21:56 +0000 Subject: [PATCH 01/14] add TIMELINE_CHECK_TARGET_THRESHOLD to descriptor.json --- .../plugin-files/descriptor/descriptor.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json index 45e15329..954c825f 100644 --- a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json +++ b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json @@ -65,10 +65,16 @@ }, { "name": "TIMELINE_CHECK_TARGET_THRESHOLD", - "description": "Specifies the number of seconds that video events can occur before or after video segment bounds. If exceeded, another attempt will be made to generate the output. Set to -1 to disable check.", + "description": "Specifies the number of seconds that video events can occur before or after video segment bounds. If exceeded, another attempt will be made to generate the output. See also the TIMELINE_CHECK_THRESHOLD_ACCEPTABLE property. Set to -1 to disable check.", "type": "INT", "defaultValue": "10" }, + { + "name": "TIMELINE_CHECK_THRESHOLD_ACCEPTABLE", + "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to -1 to disable check.", + "type": "INT", + "defaultValue": "30" + }, { "name": "TARGET_SEGMENT_LENGTH", "description": "Default segment length is 180 seconds. Set to -1 to disable segmenting the video.", From d5e6e5bfefa1c0c143fd0aafa2f79ac9771a4148 Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Thu, 22 May 2025 22:42:28 -0400 Subject: [PATCH 02/14] implement TIMELINE_CHECK_ACCEPTABLE_THRESHOLD feature --- .../__init__.py | 77 ++++++++++++++----- .../plugin-files/descriptor/descriptor.json | 2 +- .../tests/test_llama_video_summarization.py | 70 +++++++++++++++++ 3 files changed, 127 insertions(+), 22 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 49544129..2ff01f24 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -44,7 +44,7 @@ class LlamaVideoSummarizationComponent: def __init__(self): self.child_process = ChildProcess(['/llama/venv/bin/python3', '/llama/summarize_video.py', str(log.getEffectiveLevel())]) - + self.acceptable_timeline = None def get_detections_from_video(self, job: mpf.VideoJob) -> Iterable[mpf.VideoTrack]: try: @@ -62,6 +62,11 @@ def get_detections_from_video(self, job: mpf.VideoJob) -> Iterable[mpf.VideoTrac segment_stop_time = (job.stop_frame + 1) / float(job.media_properties['FPS']) job_config = _parse_properties(job.job_properties, segment_start_time) + + if job_config['timeline_check_acceptable_threshold'] < job_config['timeline_check_target_threshold']: + raise mpf.DetectionError.INVALID_PROPERTY.exception( + 'TIMELINE_CHECK_ACCEPTABLE_THRESHOLD must be >= TIMELINE_CHECK_TARGET_THRESHOLD.') + job_config['video_path'] = job.data_uri job_config['segment_start_time'] = segment_start_time job_config['segment_stop_time'] = segment_stop_time @@ -89,6 +94,7 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: max_attempts = job_config['generation_max_attempts'] timeline_check_target_threshold = job_config['timeline_check_target_threshold'] + timeline_check_acceptable_threshold = job_config['timeline_check_acceptable_threshold'] segment_start_time = job_config['segment_start_time'] segment_stop_time = job_config['segment_stop_time'] @@ -100,19 +106,20 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: if error is not None: continue - # if no error, then response_json should be valid - event_timeline = response_json['video_event_timeline'] # type: ignore - if timeline_check_target_threshold != -1: error = self._check_timeline( - timeline_check_target_threshold, attempts, max_attempts, segment_start_time, segment_stop_time, event_timeline) + timeline_check_target_threshold, timeline_check_acceptable_threshold, attempts, max_attempts, segment_start_time, segment_stop_time, response_json) if error is not None: continue break if error: - raise mpf.DetectionError.DETECTION_FAILED.exception(f'Subprocess failed: {error}') + acceptable_timeline = self._fetch_acceptable_response() + if acceptable_timeline is not None: + return acceptable_timeline + else: + raise mpf.DetectionError.DETECTION_FAILED.exception(f'Subprocess failed: {error}') # if no error, then response_json should be valid return response_json # type: ignore @@ -152,10 +159,18 @@ def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, return response_json, None - def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, - segment_start_time: float, segment_stop_time: float, event_timeline: list + def _check_timeline(self, target_threshold: float, accept_threshold: float, attempts: dict, max_attempts: int, + segment_start_time: float, segment_stop_time: float, response_json: dict ) -> Union[str, None]: + event_timeline = response_json['video_event_timeline'] # type: ignore + + # start with passing checks, then fail as secondary checks are conducted + acceptable_checks = dict( + before_seg_start = True, + after_seg_stop = True, + after_seg_start = True, + before_seg_stop = True) error = None for event in event_timeline: timestamp_start = _get_timestamp_value(event["timestamp_start"]) @@ -174,31 +189,43 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, f'{timestamp_end} < {timestamp_start}.') break - if (segment_start_time - timestamp_start) > threshold: + if (segment_start_time - timestamp_start) > target_threshold: error = (f'Timeline event start time occurs too soon before segment start time. ' - f'({segment_start_time} - {timestamp_start}) > {threshold}.') - break + f'({segment_start_time} - {timestamp_start}) > {target_threshold}.') + if (segment_start_time - timestamp_start) > accept_threshold: + acceptable_checks['before_seg_start'] = False + break - if (timestamp_end - segment_stop_time) > threshold: + if (timestamp_end - segment_stop_time) > target_threshold: error = (f'Timeline event end time occurs too late after segment stop time. ' - f'({timestamp_end} - {segment_stop_time}) > {threshold}.') - break + f'({timestamp_end} - {segment_stop_time}) > {target_threshold}.') + if (timestamp_end - segment_stop_time) > accept_threshold: + acceptable_checks['after_seg_stop'] = False + break if not error: min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), filter(lambda d: 'timestamp_start' in d, event_timeline)))) - if abs(segment_start_time - min_event_start) > threshold: + if abs(segment_start_time - min_event_start) > target_threshold: + error = (f'Min timeline event start time not close enough to segment start time. ' - f'abs({segment_start_time} - {min_event_start}) > {threshold}.') + f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.') + if abs(segment_start_time - min_event_start) > accept_threshold: + acceptable_checks['after_seg_start'] = False - if not error: max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), filter(lambda d: 'timestamp_end' in d, event_timeline)))) - if abs(max_event_end - segment_stop_time) > threshold: - error = (f'Max timeline event end time not close enough to segment stop time. ' - f'abs({max_event_end} - {segment_stop_time}) > {threshold}.') + if abs(max_event_end - segment_stop_time) > target_threshold: + if error: # keep the first encountered error + error = (f'Max timeline event end time not close enough to segment stop time. ' + f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.') + if abs(max_event_end - segment_stop_time) > target_threshold: + acceptable_checks['before_seg_stop'] = False + + if list({v for v in acceptable_checks.values()}) == [True]: + self._store_acceptable_response(response_json) if error: log.warning(error) @@ -208,6 +235,11 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, return None + def _store_acceptable_response(self, response_json: dict) -> None: + self.acceptable_timeline = response_json + + def _fetch_acceptable_response(self) -> list: + return self.acceptable_timeline def _create_segment_summary_track(self, job: mpf.VideoJob, response_json: dict) -> mpf.VideoTrack: start_frame = job.start_frame @@ -356,6 +388,8 @@ def _parse_properties(props: Mapping[str, str], segment_start_time: float) -> di props, 'GENERATION_MAX_ATTEMPTS', 5) timeline_check_target_threshold = mpf_util.get_property( props, 'TIMELINE_CHECK_TARGET_THRESHOLD', 10) + timeline_check_acceptable_threshold = mpf_util.get_property( + props, 'TIMELINE_CHECK_ACCEPTABLE_THRESHOLD', 30) generation_prompt = _read_file(generation_prompt_path) % (segment_start_time) @@ -373,7 +407,8 @@ def _parse_properties(props: Mapping[str, str], segment_start_time: float) -> di generation_json_schema = generation_json_schema, system_prompt = system_prompt, generation_max_attempts = generation_max_attempts, - timeline_check_target_threshold = timeline_check_target_threshold + timeline_check_target_threshold = timeline_check_target_threshold, + timeline_check_acceptable_threshold = timeline_check_acceptable_threshold ) diff --git a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json index 954c825f..081bf7a9 100644 --- a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json +++ b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json @@ -70,7 +70,7 @@ "defaultValue": "10" }, { - "name": "TIMELINE_CHECK_THRESHOLD_ACCEPTABLE", + "name": "TIMELINE_CHECK_ACCEPTABLE_THRESHOLD", "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to -1 to disable check.", "type": "INT", "defaultValue": "30" diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 78b84079..2ddfce1b 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -504,6 +504,76 @@ def test_timeline_integrity(self): self.assertIsNotNone(job2_results[2].frame_locations[7733]) self.assertIsNotNone(job2_results[2].frame_locations[8943]) + def test_timeline_acceptable_threshold(self): + component = LlamaVideoSummarizationComponent() + + job = mpf.VideoJob( + job_name='drone.mp4-segment-1', + data_uri=str( TEST_DATA / 'drone.mp4'), + start_frame=0, + stop_frame=5393, # 5393 + 1 = 5394 --> 179.9798 secs + job_properties=dict( + GENERATION_MAX_ATTEMPTS=2, + PROCESS_FPS=1, + MAX_FRAMES=180, + MAX_NEW_TOKENS=4096, + TIMELINE_CHECK_TARGET_THRESHOLD=10, + TIMELINE_CHECK_ACCEPTABLE_THRESHOLD=5 # must be higher than 10 + ), + media_properties=DRONE_VIDEO_PROPERTIES, + feed_forward_track=None) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps(DRONE_TIMELINE_SEGMENT_1)) + + self.assertEqual(mpf.DetectionError.INVALID_PROPERTY, cm.exception.error_code) + self.assertIn("TIMELINE_CHECK_ACCEPTABLE_THRESHOLD must be >= TIMELINE_CHECK_TARGET_THRESHOLD.", str(cm.exception)) + + job1 = mpf.VideoJob( + job_name='drone.mp4-segment-1', + data_uri=str( TEST_DATA / 'drone.mp4'), + start_frame=0, + stop_frame=5393, # 5393 + 1 = 5394 --> 179.9798 secs + job_properties=dict( + GENERATION_MAX_ATTEMPTS=2, + PROCESS_FPS=1, + MAX_FRAMES=180, + MAX_NEW_TOKENS=4096, + TIMELINE_CHECK_TARGET_THRESHOLD=10, + TIMELINE_CHECK_ACCEPTABLE_THRESHOLD=30 + ), + media_properties=DRONE_VIDEO_PROPERTIES, + feed_forward_track=None) + + DRONE_TIMELINE_SEGMENT_1["video_event_timeline"][0]["timestamp_start"] += 11.0 + DRONE_TIMELINE_SEGMENT_1["video_event_timeline"][2]["timestamp_end"] += 20.0 + job1_results = self.run_patched_job(component, job1, json.dumps(DRONE_TIMELINE_SEGMENT_1)) + self.assertEquals(4, len(job1_results)) + + + job2 = mpf.VideoJob( + job_name='drone.mp4-segment-2', + data_uri=str( TEST_DATA / 'drone.mp4'), + start_frame=5394, + stop_frame=8989, # 8989 - 5394 + 1 = 3596 --> 119.9865 secs + job_properties=dict( + GENERATION_MAX_ATTEMPTS=2, + PROCESS_FPS=1, + MAX_FRAMES=180, + MAX_NEW_TOKENS=4096, + TIMELINE_CHECK_TARGET_THRESHOLD=10, + TIMELINE_CHECK_ACCEPTABLE_THRESHOLD=30 + ), + media_properties=DRONE_VIDEO_PROPERTIES, + feed_forward_track=None) + + DRONE_TIMELINE_SEGMENT_2["video_event_timeline"].pop(0) + DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][0]["timestamp_start"] = 179.98 - 20 + DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][0]["timestamp_end"] = 178.0 + DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][-1]["timestamp_end"] = 325.0 + job2_results = self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.assertEquals(5, len(job2_results)) + if __name__ == "__main__": unittest.main(verbosity=2) \ No newline at end of file From dfed446c0d12332f517e7e303297e5e8cfd42f81 Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Fri, 23 May 2025 00:11:16 -0400 Subject: [PATCH 03/14] different failure states --- .../__init__.py | 45 +++++++---- .../tests/test_llama_video_summarization.py | 81 ++++++++++--------- 2 files changed, 73 insertions(+), 53 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 2ff01f24..6b4739ca 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -171,46 +171,48 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte after_seg_stop = True, after_seg_start = True, before_seg_stop = True) - error = None + + hard_error = None + soft_error = None for event in event_timeline: timestamp_start = _get_timestamp_value(event["timestamp_start"]) timestamp_end = _get_timestamp_value(event["timestamp_end"]) if timestamp_start < 0: - error = (f'Timeline event start time of {timestamp_start} < 0.') + hard_error = (f'Timeline event start time of {timestamp_start} < 0.') break if timestamp_end < 0: - error = (f'Timeline event end time of {timestamp_end} < 0.') + hard_error = (f'Timeline event end time of {timestamp_end} < 0.') break if timestamp_end < timestamp_start: - error = (f'Timeline event end time is less than event start time. ' + hard_error = (f'Timeline event end time is less than event start time. ' f'{timestamp_end} < {timestamp_start}.') break if (segment_start_time - timestamp_start) > target_threshold: - error = (f'Timeline event start time occurs too soon before segment start time. ' + soft_error = (f'Timeline event start time occurs too soon before segment start time. ' f'({segment_start_time} - {timestamp_start}) > {target_threshold}.') if (segment_start_time - timestamp_start) > accept_threshold: acceptable_checks['before_seg_start'] = False break if (timestamp_end - segment_stop_time) > target_threshold: - error = (f'Timeline event end time occurs too late after segment stop time. ' + soft_error = (f'Timeline event end time occurs too late after segment stop time. ' f'({timestamp_end} - {segment_stop_time}) > {target_threshold}.') if (timestamp_end - segment_stop_time) > accept_threshold: acceptable_checks['after_seg_stop'] = False break - - if not error: + + minmax_errors = [] + if not hard_error: min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), filter(lambda d: 'timestamp_start' in d, event_timeline)))) - if abs(segment_start_time - min_event_start) > target_threshold: - error = (f'Min timeline event start time not close enough to segment start time. ' - f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.') + minmax_errors.append((f'Min timeline event start time not close enough to segment start time. ' + f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.')) if abs(segment_start_time - min_event_start) > accept_threshold: acceptable_checks['after_seg_start'] = False @@ -218,14 +220,23 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte filter(lambda d: 'timestamp_end' in d, event_timeline)))) if abs(max_event_end - segment_stop_time) > target_threshold: - if error: # keep the first encountered error - error = (f'Max timeline event end time not close enough to segment stop time. ' - f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.') - if abs(max_event_end - segment_stop_time) > target_threshold: + minmax_errors.append((f'Max timeline event end time not close enough to segment stop time. ' + f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.')) + if abs(max_event_end - segment_stop_time) > accept_threshold: acceptable_checks['before_seg_stop'] = False - if list({v for v in acceptable_checks.values()}) == [True]: - self._store_acceptable_response(response_json) + if not hard_error: + if list({v for v in acceptable_checks.values()}) == [True]: + self._store_acceptable_response(response_json) + + if len(minmax_errors) > 0: + soft_error = minmax_errors.pop() + + error = None + if hard_error: + error = hard_error + elif soft_error: + error = soft_error if error: log.warning(error) diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 2ddfce1b..6c14e957 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -288,13 +288,13 @@ def test_multiple_videos(self): self.assertEquals(0, results[1].stop_frame) # (1 * 1) - 1 self.assert_first_middle_last_detections(results[1], frame_width, frame_height) - def test_invalid_timeline(self): component = LlamaVideoSummarizationComponent() job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 15000, { - "GENERATION_MAX_ATTEMPTS" : "1" + "GENERATION_MAX_ATTEMPTS" : "1", + "TIMELINE_CHECK_TARGET_THRESHOLD" : "10" }, CAT_VIDEO_PROPERTIES, {}) @@ -358,14 +358,17 @@ def test_empty_response(self): def test_timeline_integrity(self): component = LlamaVideoSummarizationComponent() - DRONE_TIMELINE_SEGMENT_1['video_event_timeline'].append({ + drone_timeline_segment_1 = DRONE_TIMELINE_SEGMENT_1.copy() + drone_timeline_segment_2 = DRONE_TIMELINE_SEGMENT_2.copy() + + drone_timeline_segment_1['video_event_timeline'].append({ "timestamp_start": 185.81, "timestamp_end": 235.77, "description": "The camera zooms in on the protesters, showing their faces and the details of their signs." }) # test min/max track frame overrides (with TIMELINE_CHECK_TARGET_THRESHOLD=-1) - DRONE_TIMELINE_SEGMENT_1["video_event_timeline"].append({ + drone_timeline_segment_1["video_event_timeline"].append({ "timestamp_start": 236.77, "timestamp_end": 179.96, "description": "The camera pans out to show the entire scene, including the fountain and the surrounding buildings." @@ -387,8 +390,8 @@ def test_timeline_integrity(self): feed_forward_track=None) # event that starts within range but ends outside of valid frames - DRONE_TIMELINE_SEGMENT_1["video_event_timeline"][2]["timestamp_end"] = 185.0 - job1_results = self.run_patched_job(component, job1, json.dumps(DRONE_TIMELINE_SEGMENT_1)) + drone_timeline_segment_1["video_event_timeline"][2]["timestamp_end"] = 185.0 + job1_results = self.run_patched_job(component, job1, json.dumps(drone_timeline_segment_1)) self.assertEquals(6, len(job1_results)) self.assertIn('SEGMENT SUMMARY', job1_results[0].detection_properties) @@ -421,69 +424,73 @@ def test_timeline_integrity(self): PROCESS_FPS=1, MAX_FRAMES=180, MAX_NEW_TOKENS=4096, - TIMELINE_CHECK_TARGET_THRESHOLD=20 + TIMELINE_CHECK_TARGET_THRESHOLD=20, + TIMELINE_CHECK_ACCEPTABLE_THRESHOLD=20 ), media_properties=DRONE_VIDEO_PROPERTIES, feed_forward_track=None) with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - self.assertIn("Timeline event start time of -45.2 < 0.", str(cm.exception)) + # TODO: rework this check + # self.assertIn("Timeline event start time of -45.2 < 0.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'].pop(0) + drone_timeline_segment_2['video_event_timeline'].pop(0) with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - self.assertIn("Timeline event start time occurs too soon before segment start time. (179.9798 - 0.0) > 20.", str(cm.exception)) + # TODO: rework this check + # self.assertIn("Timeline event start time occurs too soon before segment start time. (179.9798 - 0.0) > 20.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'].pop(0) + drone_timeline_segment_2['video_event_timeline'].pop(0) with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - self.assertIn("Timeline event end time occurs too late after segment stop time. (381.17 - 299.96633333333335) > 20.", str(cm.exception)) + # TODO: rework this check + # self.assertIn("Timeline event end time occurs too late after segment stop time. (381.17 - 299.96633333333335) > 20.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'][-1]["timestamp_end"] = 295.0 + drone_timeline_segment_2['video_event_timeline'][-1]["timestamp_end"] = 295.0 with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Timeline event end time is less than event start time. 295.0 < 299.42.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'].pop() - event1 = DRONE_TIMELINE_SEGMENT_2['video_event_timeline'].pop(0) + drone_timeline_segment_2['video_event_timeline'].pop() + event1 = drone_timeline_segment_2['video_event_timeline'].pop(0) with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Min timeline event start time not close enough to segment start time.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'].insert(0, event1) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'][1]["timestamp_end"] = -5.0 # 298.46 + drone_timeline_segment_2['video_event_timeline'].insert(0, event1) + drone_timeline_segment_2['video_event_timeline'][1]["timestamp_end"] = -5.0 # 298.46 with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Timeline event end time of -5.0 < 0.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'][1]["timestamp_end"] = 250.0 + drone_timeline_segment_2['video_event_timeline'][1]["timestamp_end"] = 250.0 with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Max timeline event end time not close enough to segment stop time.", str(cm.exception)) - DRONE_TIMELINE_SEGMENT_2['video_event_timeline'][1]["timestamp_end"] = 298.46 - job2_results = self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + drone_timeline_segment_2['video_event_timeline'][1]["timestamp_end"] = 298.46 + job2_results = self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEquals(3, len(job2_results)) self.assertIn('SEGMENT SUMMARY', job2_results[0].detection_properties) @@ -506,6 +513,8 @@ def test_timeline_integrity(self): def test_timeline_acceptable_threshold(self): component = LlamaVideoSummarizationComponent() + drone_timeline_segment_1 = DRONE_TIMELINE_SEGMENT_1.copy() + drone_timeline_segment_2 = DRONE_TIMELINE_SEGMENT_2.copy() job = mpf.VideoJob( job_name='drone.mp4-segment-1', @@ -524,7 +533,7 @@ def test_timeline_acceptable_threshold(self): feed_forward_track=None) with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job, json.dumps(DRONE_TIMELINE_SEGMENT_1)) + self.run_patched_job(component, job, json.dumps(drone_timeline_segment_1)) self.assertEqual(mpf.DetectionError.INVALID_PROPERTY, cm.exception.error_code) self.assertIn("TIMELINE_CHECK_ACCEPTABLE_THRESHOLD must be >= TIMELINE_CHECK_TARGET_THRESHOLD.", str(cm.exception)) @@ -545,9 +554,9 @@ def test_timeline_acceptable_threshold(self): media_properties=DRONE_VIDEO_PROPERTIES, feed_forward_track=None) - DRONE_TIMELINE_SEGMENT_1["video_event_timeline"][0]["timestamp_start"] += 11.0 - DRONE_TIMELINE_SEGMENT_1["video_event_timeline"][2]["timestamp_end"] += 20.0 - job1_results = self.run_patched_job(component, job1, json.dumps(DRONE_TIMELINE_SEGMENT_1)) + drone_timeline_segment_1["video_event_timeline"][0]["timestamp_start"] += 11.0 + drone_timeline_segment_1["video_event_timeline"][2]["timestamp_end"] += 20.0 + job1_results = self.run_patched_job(component, job1, json.dumps(drone_timeline_segment_1)) self.assertEquals(4, len(job1_results)) @@ -567,11 +576,11 @@ def test_timeline_acceptable_threshold(self): media_properties=DRONE_VIDEO_PROPERTIES, feed_forward_track=None) - DRONE_TIMELINE_SEGMENT_2["video_event_timeline"].pop(0) - DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][0]["timestamp_start"] = 179.98 - 20 - DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][0]["timestamp_end"] = 178.0 - DRONE_TIMELINE_SEGMENT_2["video_event_timeline"][-1]["timestamp_end"] = 325.0 - job2_results = self.run_patched_job(component, job2, json.dumps(DRONE_TIMELINE_SEGMENT_2)) + drone_timeline_segment_2["video_event_timeline"].pop(0) + drone_timeline_segment_2["video_event_timeline"][0]["timestamp_start"] = 179.98 - 20 + drone_timeline_segment_2["video_event_timeline"][0]["timestamp_end"] = 178.0 + drone_timeline_segment_2["video_event_timeline"][-1]["timestamp_end"] = 325.0 + job2_results = self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEquals(5, len(job2_results)) From ec6153ddbd7f7cc3184a8ca0ff89df30031cbd32 Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 28 May 2025 16:52:58 +0000 Subject: [PATCH 04/14] Validate timestamps. --- .../__init__.py | 66 ++++++++++--------- .../tests/test_llama_video_summarization.py | 56 +++++++++++++++- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 49544129..b3f90868 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -120,36 +120,38 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, response: str ) -> Tuple[Union[dict, None], Union[str, None]]: + error_msg = None response_json = None if not response: - error = 'Empty response.' - log.warning(error) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return None, error - - try: - response_json = json.loads(response) - except ValueError as ve: - error = 'Response is not valid JSON.' - log.warning(error) - log.warning(str(ve)) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error - - try: - validate(response_json, schema_json) - except ValidationError as ve: - error = 'Response JSON is not in the desired format.' - log.warning(error) - log.warning(str(ve)) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error + error_msg = 'Empty response.' + + if not error_msg: + try: + response_json = json.loads(response) + except ValueError as ve: + error_msg = f'Response is not valid JSON. {str(ve)}' + + if not error_msg and response_json: + try: + validate(response_json, schema_json) + except ValidationError as ve: + error_msg = f'Response JSON is not in the desired format. {str(ve)}' - return response_json, None + if not error_msg and response_json: + try: + event_timeline = response_json['video_event_timeline'] + for event in event_timeline: + # update values for later use + event["timestamp_start"] = _get_timestamp_value(event["timestamp_start"]) + event["timestamp_end"] = _get_timestamp_value(event["timestamp_end"]) + except ValueError as ve: + error_msg = f'Response JSON is not in the desired format. {str(ve)}' + + log.warning(error_msg) + log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') + attempts['base'] += 1 + return response_json, error_msg def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, @@ -158,8 +160,8 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, error = None for event in event_timeline: - timestamp_start = _get_timestamp_value(event["timestamp_start"]) - timestamp_end = _get_timestamp_value(event["timestamp_end"]) + timestamp_start = event["timestamp_start"] + timestamp_end = event["timestamp_end"] if timestamp_start < 0: error = (f'Timeline event start time of {timestamp_start} < 0.') @@ -185,7 +187,7 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, break if not error: - min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), + min_event_start = min(list(map(lambda d: d.get('timestamp_start'), filter(lambda d: 'timestamp_start' in d, event_timeline)))) if abs(segment_start_time - min_event_start) > threshold: @@ -193,7 +195,7 @@ def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, f'abs({segment_start_time} - {min_event_start}) > {threshold}.') if not error: - max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), + max_event_end = max(list(map(lambda d: d.get('timestamp_end'), filter(lambda d: 'timestamp_end' in d, event_timeline)))) if abs(max_event_end - segment_stop_time) > threshold: @@ -263,8 +265,8 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf for event in response_json['video_event_timeline']: # get offset start/stop times in milliseconds - event_start_time = int(_get_timestamp_value(event['timestamp_start']) * 1000) - event_stop_time = int(_get_timestamp_value(event['timestamp_end']) * 1000) + event_start_time = int(event['timestamp_start'] * 1000) + event_stop_time = int(event['timestamp_end'] * 1000) offset_start_frame = int((event_start_time * video_fps) / 1000) offset_stop_frame = int((event_stop_time * video_fps) / 1000) - 1 diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 78b84079..95f61537 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -69,7 +69,7 @@ }, { "timestamp_start": "5.0", - "timestamp_end": "6.8", + "timestamp_end": "6.8s", "description": "The cat looks back at the camera and then walks away." } ] @@ -340,6 +340,59 @@ def test_invalid_json_response(self): self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("not valid JSON", str(cm.exception)) + + def test_schema_check(self): + component = LlamaVideoSummarizationComponent() + + job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 171, + { + "GENERATION_MAX_ATTEMPTS" : "1" + }, + CAT_VIDEO_PROPERTIES, {}) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps( + { + "video_summary": "This is a video of a cat.", + "video_event_timeline": [ + { + "timestamp_start": "0.00", + "bad": "8.04", + "description": "The cat is sitting on the cobblestone street, looking around." + } + ] + })) # don't care about results + + self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) + self.assertIn("'timestamp_end' is a required property", str(cm.exception)) + + + def test_invalid_timestamp(self): + component = LlamaVideoSummarizationComponent() + + job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 171, + { + "GENERATION_MAX_ATTEMPTS" : "1" + }, + CAT_VIDEO_PROPERTIES, {}) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps( + { + "video_summary": "This is a video of a cat.", + "video_event_timeline": [ + { + "timestamp_start": "7:12", + "timestamp_end": "8:04", + "description": "The cat is sitting on the cobblestone street, looking around." + } + ] + })) # don't care about results + + self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) + self.assertIn("could not convert string to float", str(cm.exception)) + + def test_empty_response(self): component = LlamaVideoSummarizationComponent() @@ -355,6 +408,7 @@ def test_empty_response(self): self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) self.assertIn("Empty response", str(cm.exception)) + def test_timeline_integrity(self): component = LlamaVideoSummarizationComponent() From 14f30adc27421da3157b1e5fe6be0dbe3df8ecfb Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 28 May 2025 17:06:22 +0000 Subject: [PATCH 05/14] Fix bug. --- .../__init__.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index b3f90868..3cceba56 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -120,25 +120,25 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, response: str ) -> Tuple[Union[dict, None], Union[str, None]]: - error_msg = None + error = None response_json = None if not response: - error_msg = 'Empty response.' + error = 'Empty response.' - if not error_msg: + if not error: try: response_json = json.loads(response) except ValueError as ve: - error_msg = f'Response is not valid JSON. {str(ve)}' + error = f'Response is not valid JSON. {str(ve)}' - if not error_msg and response_json: + if not error and response_json: try: validate(response_json, schema_json) except ValidationError as ve: - error_msg = f'Response JSON is not in the desired format. {str(ve)}' + error = f'Response JSON is not in the desired format. {str(ve)}' - if not error_msg and response_json: + if not error and response_json: try: event_timeline = response_json['video_event_timeline'] for event in event_timeline: @@ -146,12 +146,14 @@ def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, event["timestamp_start"] = _get_timestamp_value(event["timestamp_start"]) event["timestamp_end"] = _get_timestamp_value(event["timestamp_end"]) except ValueError as ve: - error_msg = f'Response JSON is not in the desired format. {str(ve)}' + error = f'Response JSON is not in the desired format. {str(ve)}' - log.warning(error_msg) - log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') - attempts['base'] += 1 - return response_json, error_msg + if error: + log.warning(error) + log.warning(f'Failed {attempts["base"] + 1} of {max_attempts} base attempts.') + attempts['base'] += 1 + + return response_json, error def _check_timeline(self, threshold: float, attempts: dict, max_attempts: int, From 260ed4cd92bbab7bade0e2f8d629239624ba4447 Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Tue, 8 Jul 2025 00:12:41 +0000 Subject: [PATCH 06/14] refactor tests --- .../tests/test_llama_video_summarization.py | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 6c14e957..da3573bd 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -26,6 +26,7 @@ from __future__ import annotations +import copy import json import logging import os @@ -358,8 +359,8 @@ def test_empty_response(self): def test_timeline_integrity(self): component = LlamaVideoSummarizationComponent() - drone_timeline_segment_1 = DRONE_TIMELINE_SEGMENT_1.copy() - drone_timeline_segment_2 = DRONE_TIMELINE_SEGMENT_2.copy() + drone_timeline_segment_1 = copy.deepcopy(DRONE_TIMELINE_SEGMENT_1) + drone_timeline_segment_2 = copy.deepcopy(DRONE_TIMELINE_SEGMENT_2) drone_timeline_segment_1['video_event_timeline'].append({ "timestamp_start": 185.81, @@ -434,27 +435,10 @@ def test_timeline_integrity(self): self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - # TODO: rework this check - # self.assertIn("Timeline event start time of -45.2 < 0.", str(cm.exception)) + self.assertIn("Timeline event start time of -45.2 < 0.", str(cm.exception)) drone_timeline_segment_2['video_event_timeline'].pop(0) - - with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) - - self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - # TODO: rework this check - # self.assertIn("Timeline event start time occurs too soon before segment start time. (179.9798 - 0.0) > 20.", str(cm.exception)) - drone_timeline_segment_2['video_event_timeline'].pop(0) - - with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job2, json.dumps(drone_timeline_segment_2)) - - self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - # TODO: rework this check - # self.assertIn("Timeline event end time occurs too late after segment stop time. (381.17 - 299.96633333333335) > 20.", str(cm.exception)) - drone_timeline_segment_2['video_event_timeline'][-1]["timestamp_end"] = 295.0 with self.assertRaises(mpf.DetectionException) as cm: @@ -513,8 +497,8 @@ def test_timeline_integrity(self): def test_timeline_acceptable_threshold(self): component = LlamaVideoSummarizationComponent() - drone_timeline_segment_1 = DRONE_TIMELINE_SEGMENT_1.copy() - drone_timeline_segment_2 = DRONE_TIMELINE_SEGMENT_2.copy() + drone_timeline_segment_1 = copy.deepcopy(DRONE_TIMELINE_SEGMENT_1) + drone_timeline_segment_2 = copy.deepcopy(DRONE_TIMELINE_SEGMENT_2) job = mpf.VideoJob( job_name='drone.mp4-segment-1', From aa885029f76fda5cf81ebdaa0ffe8a8df7a8f7f8 Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Fri, 19 Sep 2025 20:14:58 +0000 Subject: [PATCH 07/14] raise exception on invalid timestamps --- .../__init__.py | 6 ++++- .../tests/test_llama_video_summarization.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 6b4739ca..59fa0ed2 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -31,6 +31,7 @@ import pickle import socket import subprocess +import re from jsonschema import validate, ValidationError from typing import Any, Iterable, List, Mapping, Tuple, Union @@ -376,7 +377,10 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf def _get_timestamp_value(seconds: Any) -> float: if isinstance(seconds, str): - secval = float(seconds.replace('s', '')) + if re.match(r"\s*\d+(\.\d*)?\s*[Ss]", seconds): + secval = float(re.sub('s', '', seconds, flags=re.IGNORECASE)) + else: + raise mpf.DetectionError.DETECTION_FAILED.exception(f'Invalid timestamp: {seconds}') else: secval = float(seconds) return secval diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index da3573bd..4d2fd7be 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -325,6 +325,29 @@ def test_invalid_timeline(self): self.assertIn("cat", results[0].detection_properties["TEXT"]) + def test_invalid_timestamp_format(self): + component = LlamaVideoSummarizationComponent() + + cat_timeline = copy.deepcopy(CAT_TIMELINE["video_event_timeline"]) + cat_timeline[0]["timestamp_start"] = "7:12:03.234" + + job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 15000, + { + "GENERATION_MAX_ATTEMPTS" : "1", + "TIMELINE_CHECK_TARGET_THRESHOLD" : "10" + }, + CAT_VIDEO_PROPERTIES, {}) + + with self.assertRaises(mpf.DetectionException) as cm: + self.run_patched_job(component, job, json.dumps( + { + "video_summary": "This is a video of a cat.", + "video_event_timeline": cat_timeline + })) # don't care about results + + self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) + self.assertIn("Invalid timestamp: ", str(cm.exception)) + def test_invalid_json_response(self): component = LlamaVideoSummarizationComponent() From b93087770200ead712a51b1ec7309e7c13fab42c Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Fri, 19 Sep 2025 21:27:58 +0000 Subject: [PATCH 08/14] update test assertion --- .../tests/test_llama_video_summarization.py | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py index 83f62825..a917a62e 100644 --- a/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py +++ b/python/LlamaVideoSummarization/tests/test_llama_video_summarization.py @@ -325,29 +325,6 @@ def test_invalid_timeline(self): self.assertIn("cat", results[0].detection_properties["TEXT"]) - def test_invalid_timestamp_format(self): - component = LlamaVideoSummarizationComponent() - - cat_timeline = copy.deepcopy(CAT_TIMELINE["video_event_timeline"]) - cat_timeline[0]["timestamp_start"] = "7:12:03.234" - - job = mpf.VideoJob('cat job', str(TEST_DATA / 'cat.mp4'), 0, 15000, - { - "GENERATION_MAX_ATTEMPTS" : "1", - "TIMELINE_CHECK_TARGET_THRESHOLD" : "10" - }, - CAT_VIDEO_PROPERTIES, {}) - - with self.assertRaises(mpf.DetectionException) as cm: - self.run_patched_job(component, job, json.dumps( - { - "video_summary": "This is a video of a cat.", - "video_event_timeline": cat_timeline - })) # don't care about results - - self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - self.assertIn("Invalid timestamp: ", str(cm.exception)) - def test_invalid_json_response(self): component = LlamaVideoSummarizationComponent() @@ -414,7 +391,7 @@ def test_invalid_timestamp(self): })) # don't care about results self.assertEqual(mpf.DetectionError.DETECTION_FAILED, cm.exception.error_code) - self.assertIn("could not convert string to float", str(cm.exception)) + self.assertIn("Invalid timestamp: ", str(cm.exception)) def test_empty_response(self): From d588a08ffaa4feb4f5abd1d5a471a5d8032d443c Mon Sep 17 00:00:00 2001 From: Ben Huyck Date: Fri, 19 Sep 2025 21:28:35 +0000 Subject: [PATCH 09/14] fix bug in timestamp check pattern --- .../llama_video_summarization_component/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 49a13729..e5f1ea54 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -381,7 +381,7 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf def _get_timestamp_value(seconds: Any) -> float: if isinstance(seconds, str): - if re.match(r"\s*\d+(\.\d*)?\s*[Ss]", seconds): + if re.match(r"\s*\d+(\.\d*)?\s*[Ss]?", seconds): secval = float(re.sub('s', '', seconds, flags=re.IGNORECASE)) else: raise mpf.DetectionError.DETECTION_FAILED.exception(f'Invalid timestamp: {seconds}') From 69262ebb379e1abefa927a109fcd9276a4461a3b Mon Sep 17 00:00:00 2001 From: jrobble Date: Mon, 29 Sep 2025 12:15:35 -0400 Subject: [PATCH 10/14] Address PR comments. --- .../__init__.py | 69 +++++++------------ .../plugin-files/descriptor/descriptor.json | 2 +- 2 files changed, 27 insertions(+), 44 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index e5f1ea54..0076fd22 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -99,7 +99,8 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: segment_start_time = job_config['segment_start_time'] segment_stop_time = job_config['segment_stop_time'] - response_json = {} + response_json = None + acceptable_json = None error = None while max(attempts.values()) < max_attempts: response = self.child_process.send_job_get_response(job_config) @@ -108,21 +109,24 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: continue if timeline_check_target_threshold != -1: - error = self._check_timeline( - timeline_check_target_threshold, timeline_check_acceptable_threshold, attempts, max_attempts, segment_start_time, segment_stop_time, response_json) + acceptable, error = self._check_timeline( + timeline_check_target_threshold, timeline_check_acceptable_threshold, + attempts, max_attempts, segment_start_time, segment_stop_time, response_json) + if acceptable: + acceptable_json = response_json if error is not None: continue break if error: - acceptable_timeline = self._fetch_acceptable_response() - if acceptable_timeline is not None: - return acceptable_timeline + if acceptable_json is not None: + log.info('Couldn\'t satisfy target threshold. Falling back to response that satisfies acceptable threshold.') + return acceptable_json else: raise mpf.DetectionError.DETECTION_FAILED.exception(f'Subprocess failed: {error}') - # if no error, then response_json should be valid + # if no error, then response_json should be valid and meet target criteria return response_json # type: ignore @@ -166,16 +170,13 @@ def _check_response(self, attempts: dict, max_attempts: int, schema_json: dict, def _check_timeline(self, target_threshold: float, accept_threshold: float, attempts: dict, max_attempts: int, segment_start_time: float, segment_stop_time: float, response_json: dict - ) -> Union[str, None]: + ) -> Tuple[bool, Union[str, None]]: event_timeline = response_json['video_event_timeline'] # type: ignore - # start with passing checks, then fail as secondary checks are conducted acceptable_checks = dict( - before_seg_start = True, - after_seg_stop = True, - after_seg_start = True, - before_seg_stop = True) + near_seg_start = False, + near_seg_stop = False) hard_error = None soft_error = None @@ -195,31 +196,17 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte hard_error = (f'Timeline event end time is less than event start time. ' f'{timestamp_end} < {timestamp_start}.') break - - if (segment_start_time - timestamp_start) > target_threshold: - soft_error = (f'Timeline event start time occurs too soon before segment start time. ' - f'({segment_start_time} - {timestamp_start}) > {target_threshold}.') - if (segment_start_time - timestamp_start) > accept_threshold: - acceptable_checks['before_seg_start'] = False - break - - if (timestamp_end - segment_stop_time) > target_threshold: - soft_error = (f'Timeline event end time occurs too late after segment stop time. ' - f'({timestamp_end} - {segment_stop_time}) > {target_threshold}.') - if (timestamp_end - segment_stop_time) > accept_threshold: - acceptable_checks['after_seg_stop'] = False - break minmax_errors = [] if not hard_error: min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), filter(lambda d: 'timestamp_start' in d, event_timeline)))) + if abs(segment_start_time - min_event_start) > target_threshold: - minmax_errors.append((f'Min timeline event start time not close enough to segment start time. ' f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.')) - if abs(segment_start_time - min_event_start) > accept_threshold: - acceptable_checks['after_seg_start'] = False + acceptable_checks['near_seg_start'] = \ + not (abs(segment_start_time - min_event_start) > accept_threshold) max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), filter(lambda d: 'timestamp_end' in d, event_timeline)))) @@ -227,12 +214,10 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte if abs(max_event_end - segment_stop_time) > target_threshold: minmax_errors.append((f'Max timeline event end time not close enough to segment stop time. ' f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.')) - if abs(max_event_end - segment_stop_time) > accept_threshold: - acceptable_checks['before_seg_stop'] = False + acceptable_checks['near_seg_stop'] = \ + not (abs(max_event_end - segment_stop_time) > accept_threshold) - if not hard_error: - if list({v for v in acceptable_checks.values()}) == [True]: - self._store_acceptable_response(response_json) + acceptable = not hard_error and all(acceptable_checks.values()) if len(minmax_errors) > 0: soft_error = minmax_errors.pop() @@ -247,15 +232,9 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte log.warning(error) log.warning(f'Failed {attempts["timeline"] + 1} of {max_attempts} timeline attempts.') attempts['timeline'] += 1 - return error - - return None - - def _store_acceptable_response(self, response_json: dict) -> None: - self.acceptable_timeline = response_json + + return acceptable, error - def _fetch_acceptable_response(self) -> list: - return self.acceptable_timeline def _create_segment_summary_track(self, job: mpf.VideoJob, response_json: dict) -> mpf.VideoTrack: start_frame = job.start_frame @@ -379,6 +358,7 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf log.info('Processing complete. Video segment %s summarized in %d tracks.' % (segment_id, len(tracks))) return tracks + def _get_timestamp_value(seconds: Any) -> float: if isinstance(seconds, str): if re.match(r"\s*\d+(\.\d*)?\s*[Ss]?", seconds): @@ -389,6 +369,7 @@ def _get_timestamp_value(seconds: Any) -> float: secval = float(seconds) return secval + def _parse_properties(props: Mapping[str, str], segment_start_time: float) -> dict: process_fps = mpf_util.get_property( props, 'PROCESS_FPS', 1) @@ -454,6 +435,7 @@ def __init__(self, start_cmd: List[str]): env=env) self._socket = parent_socket.makefile('rwb') + def __del__(self): print("Terminating subprocess...") self._socket.close() @@ -461,6 +443,7 @@ def __del__(self): self._proc.wait() print("Subprocess terminated") + def send_job_get_response(self, config: dict): job_bytes = pickle.dumps(config) self._socket.write(len(job_bytes).to_bytes(4, 'little')) diff --git a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json index 081bf7a9..ea68af8d 100644 --- a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json +++ b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json @@ -71,7 +71,7 @@ }, { "name": "TIMELINE_CHECK_ACCEPTABLE_THRESHOLD", - "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to -1 to disable check.", + "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_TARGET_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to -1 to disable check.", "type": "INT", "defaultValue": "30" }, From 807ad3e5e5aa7cbc4c147b33b101a41ae12836ab Mon Sep 17 00:00:00 2001 From: jrobble Date: Mon, 29 Sep 2025 16:15:04 -0400 Subject: [PATCH 11/14] Remove self.acceptable_timeline. --- .../llama_video_summarization_component/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 0076fd22..863e4d1f 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -45,7 +45,6 @@ class LlamaVideoSummarizationComponent: def __init__(self): self.child_process = ChildProcess(['/llama/venv/bin/python3', '/llama/summarize_video.py', str(log.getEffectiveLevel())]) - self.acceptable_timeline = None def get_detections_from_video(self, job: mpf.VideoJob) -> Iterable[mpf.VideoTrack]: try: From 610e9709f8ab75cd34bedb6dda5ea8053f6ebff7 Mon Sep 17 00:00:00 2001 From: jrobble Date: Tue, 30 Sep 2025 17:58:14 +0000 Subject: [PATCH 12/14] Fix unit tests. --- .../__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 863e4d1f..9350004e 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -34,7 +34,7 @@ import re from jsonschema import validate, ValidationError -from typing import Any, Iterable, List, Mapping, Tuple, Union +from typing import Any, cast, Iterable, List, Mapping, Tuple, Union import mpf_component_api as mpf import mpf_component_util as mpf_util @@ -110,7 +110,7 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: if timeline_check_target_threshold != -1: acceptable, error = self._check_timeline( timeline_check_target_threshold, timeline_check_acceptable_threshold, - attempts, max_attempts, segment_start_time, segment_stop_time, response_json) + attempts, max_attempts, segment_start_time, segment_stop_time, cast(dict, response_json)) if acceptable: acceptable_json = response_json if error is not None: @@ -204,8 +204,9 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte if abs(segment_start_time - min_event_start) > target_threshold: minmax_errors.append((f'Min timeline event start time not close enough to segment start time. ' f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.')) - acceptable_checks['near_seg_start'] = \ - not (abs(segment_start_time - min_event_start) > accept_threshold) + + acceptable_checks['near_seg_start'] = \ + not (abs(segment_start_time - min_event_start) > accept_threshold) max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), filter(lambda d: 'timestamp_end' in d, event_timeline)))) @@ -213,8 +214,9 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte if abs(max_event_end - segment_stop_time) > target_threshold: minmax_errors.append((f'Max timeline event end time not close enough to segment stop time. ' f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.')) - acceptable_checks['near_seg_stop'] = \ - not (abs(max_event_end - segment_stop_time) > accept_threshold) + + acceptable_checks['near_seg_stop'] = \ + not (abs(max_event_end - segment_stop_time) > accept_threshold) acceptable = not hard_error and all(acceptable_checks.values()) @@ -360,7 +362,7 @@ def _create_tracks(self, job: mpf.VideoJob, response_json: dict) -> Iterable[mpf def _get_timestamp_value(seconds: Any) -> float: if isinstance(seconds, str): - if re.match(r"\s*\d+(\.\d*)?\s*[Ss]?", seconds): + if re.match(r"^\s*\d+(\.\d*)?\s*[Ss]?$", seconds): secval = float(re.sub('s', '', seconds, flags=re.IGNORECASE)) else: raise mpf.DetectionError.DETECTION_FAILED.exception(f'Invalid timestamp: {seconds}') From f73fe905375d1572267b38a22e178547e00c6545 Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 1 Oct 2025 18:23:40 +0000 Subject: [PATCH 13/14] Ensure target threshold is set when acceptable threshold is set. --- .../__init__.py | 26 +++++++++++-------- .../plugin-files/descriptor/descriptor.json | 4 +-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index 9350004e..f1638fac 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -63,6 +63,11 @@ def get_detections_from_video(self, job: mpf.VideoJob) -> Iterable[mpf.VideoTrac job_config = _parse_properties(job.job_properties, segment_start_time) + if job_config['timeline_check_target_threshold'] < 0 and \ + job_config['timeline_check_acceptable_threshold'] >= 0: + raise mpf.DetectionError.INVALID_PROPERTY.exception( + 'TIMELINE_CHECK_TARGET_THRESHOLD must be >= 0 when TIMELINE_CHECK_ACCEPTABLE_THRESHOLD >= 0.') + if job_config['timeline_check_acceptable_threshold'] < job_config['timeline_check_target_threshold']: raise mpf.DetectionError.INVALID_PROPERTY.exception( 'TIMELINE_CHECK_ACCEPTABLE_THRESHOLD must be >= TIMELINE_CHECK_TARGET_THRESHOLD.') @@ -107,7 +112,7 @@ def _get_response_from_subprocess(self, job_config: dict) -> dict: if error is not None: continue - if timeline_check_target_threshold != -1: + if timeline_check_target_threshold >= 0: acceptable, error = self._check_timeline( timeline_check_target_threshold, timeline_check_acceptable_threshold, attempts, max_attempts, segment_start_time, segment_stop_time, cast(dict, response_json)) @@ -201,22 +206,21 @@ def _check_timeline(self, target_threshold: float, accept_threshold: float, atte min_event_start = min(list(map(lambda d: _get_timestamp_value(d.get('timestamp_start')), filter(lambda d: 'timestamp_start' in d, event_timeline)))) - if abs(segment_start_time - min_event_start) > target_threshold: - minmax_errors.append((f'Min timeline event start time not close enough to segment start time. ' - f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.')) - - acceptable_checks['near_seg_start'] = \ - not (abs(segment_start_time - min_event_start) > accept_threshold) - max_event_end = max(list(map(lambda d: _get_timestamp_value(d.get('timestamp_end')), filter(lambda d: 'timestamp_end' in d, event_timeline)))) + if abs(segment_start_time - min_event_start) > target_threshold: + minmax_errors.append((f'Min timeline event start time not close enough to segment start time. ' + f'abs({segment_start_time} - {min_event_start}) > {target_threshold}.')) + if abs(max_event_end - segment_stop_time) > target_threshold: minmax_errors.append((f'Max timeline event end time not close enough to segment stop time. ' - f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.')) + f'abs({max_event_end} - {segment_stop_time}) > {target_threshold}.')) + + if accept_threshold >= 0: + acceptable_checks['near_seg_start'] = abs(segment_start_time - min_event_start) <= accept_threshold - acceptable_checks['near_seg_stop'] = \ - not (abs(max_event_end - segment_stop_time) > accept_threshold) + acceptable_checks['near_seg_stop'] = abs(max_event_end - segment_stop_time) <= accept_threshold acceptable = not hard_error and all(acceptable_checks.values()) diff --git a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json index ea68af8d..55f7943a 100644 --- a/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json +++ b/python/LlamaVideoSummarization/plugin-files/descriptor/descriptor.json @@ -65,13 +65,13 @@ }, { "name": "TIMELINE_CHECK_TARGET_THRESHOLD", - "description": "Specifies the number of seconds that video events can occur before or after video segment bounds. If exceeded, another attempt will be made to generate the output. See also the TIMELINE_CHECK_THRESHOLD_ACCEPTABLE property. Set to -1 to disable check.", + "description": "Specifies the number of seconds that video events can occur before or after video segment bounds. If exceeded, another attempt will be made to generate the output. See also the TIMELINE_CHECK_THRESHOLD_ACCEPTABLE property. Set to < 0 to disable check (e.g. -1).", "type": "INT", "defaultValue": "10" }, { "name": "TIMELINE_CHECK_ACCEPTABLE_THRESHOLD", - "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_TARGET_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to -1 to disable check.", + "description": "A secondary timeline validation threshold, in seconds that specifies the number of seconds video events can occur before or after video segment bounds, which will result in an \"acceptable\" timeline. Additional attempts will be made to generate a timeline within the \"desired\" range of TIMELINE_CHECK_TARGET_THRESHOLD, until GENERATION_MAX_ATTEMPTS is reached, after which the \"acceptable\" timeline is returned, or the component responds with an error. Set to < 0 to disable check (e.g. -1).", "type": "INT", "defaultValue": "30" }, From 20980e76fcf5608ed25df7aae68b66e49229f910 Mon Sep 17 00:00:00 2001 From: jrobble Date: Wed, 1 Oct 2025 18:33:50 +0000 Subject: [PATCH 14/14] Change exception to warning. --- .../llama_video_summarization_component/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py index f1638fac..2b749651 100644 --- a/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py +++ b/python/LlamaVideoSummarization/llama_video_summarization_component/__init__.py @@ -65,8 +65,7 @@ def get_detections_from_video(self, job: mpf.VideoJob) -> Iterable[mpf.VideoTrac if job_config['timeline_check_target_threshold'] < 0 and \ job_config['timeline_check_acceptable_threshold'] >= 0: - raise mpf.DetectionError.INVALID_PROPERTY.exception( - 'TIMELINE_CHECK_TARGET_THRESHOLD must be >= 0 when TIMELINE_CHECK_ACCEPTABLE_THRESHOLD >= 0.') + log.warning('TIMELINE_CHECK_ACCEPTABLE_THRESHOLD will be ignored since TIMELINE_CHECK_TARGET_THRESHOLD < 0.') if job_config['timeline_check_acceptable_threshold'] < job_config['timeline_check_target_threshold']: raise mpf.DetectionError.INVALID_PROPERTY.exception(