From aec333db7d0651fb8a1615424be856e6cd75e001 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 26 Mar 2024 18:07:55 +0200 Subject: [PATCH] Fix two problems with the task segmentation algorithm (#7681) 1. When a task has non-zero overlap and exactly as many frames as needed to create 1 or more complete segments, the current algorithm generates a redundant segment at the end. For example, if size is 5, segment size is 3, and overlap is 1, it generates segments (0, 2), (2, 4), and (4, 4). The algorithm attempts to compensate for this, but it only works in the case where the segment size is unspecified (and defaults to the total size). Update the algorithm to handle this correctly in the general case. 2. The algorithm selects a default overlap size of 5 if the media file is a video. However, this might not be a valid value if the task has a very small segment size. In this case, a range of undesirable behaviors may occur, depending on the segment size: * segments getting generated such that more than 2 segments cover a single frame; * task creation crashing with an exception; * a task being created with no segments at all. Fix this by clamping the default overlap size the same way as a user-specified one. Fixes #7675. --- .../20240326_133323_roman_segmentation_fix.md | 9 +++ cvat/apps/engine/task.py | 18 ++--- tests/python/requirements.txt | 1 + tests/python/rest_api/test_tasks.py | 69 ++++++++++++++++++- tests/python/shared/utils/helpers.py | 22 ++++++ 5 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 changelog.d/20240326_133323_roman_segmentation_fix.md diff --git a/changelog.d/20240326_133323_roman_segmentation_fix.md b/changelog.d/20240326_133323_roman_segmentation_fix.md new file mode 100644 index 00000000000..bc9e76ee439 --- /dev/null +++ b/changelog.d/20240326_133323_roman_segmentation_fix.md @@ -0,0 +1,9 @@ +### Fixed + +- Fixed an invalid default overlap size being selected for video tasks + with small segments + () + +- Fixed redundant jobs being created for tasks with non-zero overlap + in certain cases + () diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 7e73e1f5234..7f2d6471fd3 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -6,7 +6,6 @@ import itertools import fnmatch import os -import sys from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Union, Iterable from rest_framework.serializers import ValidationError import rq @@ -132,23 +131,18 @@ def _segments(): data_size = db_task.data.size segment_size = db_task.segment_size - segment_step = segment_size if segment_size == 0 or segment_size > data_size: segment_size = data_size - # Segment step must be more than segment_size + overlap in single-segment tasks - # Otherwise a task contains an extra segment - segment_step = sys.maxsize - - overlap = 5 if db_task.mode == 'interpolation' else 0 - if db_task.overlap is not None: - overlap = min(db_task.overlap, segment_size // 2) - - segment_step -= overlap + overlap = min( + db_task.overlap if db_task.overlap is not None + else 5 if db_task.mode == 'interpolation' else 0, + segment_size // 2, + ) segments = ( SegmentParams(start_frame, min(start_frame + segment_size - 1, data_size - 1)) - for start_frame in range(0, data_size, segment_step) + for start_frame in range(0, data_size - overlap, segment_size - overlap) ) return SegmentsParams(segments, segment_size, overlap) diff --git a/tests/python/requirements.txt b/tests/python/requirements.txt index ba7d9cac8e2..d2cf925ed15 100644 --- a/tests/python/requirements.txt +++ b/tests/python/requirements.txt @@ -1,3 +1,4 @@ +av==12.0.0 pytest==6.2.5 pytest-cases==3.6.13 pytest-timeout==2.1.0 diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 19af43530aa..e56d85f3109 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -40,7 +40,12 @@ patch_method, post_method, ) -from shared.utils.helpers import generate_image_file, generate_image_files, generate_manifest +from shared.utils.helpers import ( + generate_image_file, + generate_image_files, + generate_manifest, + generate_video_file, +) from .utils import ( CollectionSimpleFilterTestBase, @@ -801,6 +806,68 @@ def test_can_create_task_with_defined_start_and_stop_frames(self): (task, _) = api_client.tasks_api.retrieve(task_id) assert task.size == 4 + def test_default_overlap_for_small_segment_size(self): + task_spec = { + "name": f"test {self._USERNAME} with default overlap and small segment_size", + "labels": [{"name": "car"}], + "segment_size": 5, + } + + task_data = { + "image_quality": 75, + "client_files": [generate_video_file(8)], + } + + task_id, _ = create_task(self._USERNAME, task_spec, task_data) + + # check task size + with make_api_client(self._USERNAME) as api_client: + paginated_job_list, _ = api_client.jobs_api.list(task_id=task_id) + + jobs = paginated_job_list.results + jobs.sort(key=lambda job: job.start_frame) + + assert len(jobs) == 2 + # overlap should be 2 frames (frames 3 & 4) + assert jobs[0].start_frame == 0 + assert jobs[0].stop_frame == 4 + assert jobs[1].start_frame == 3 + assert jobs[1].stop_frame == 7 + + @pytest.mark.parametrize( + "size,expected_segments", + [ + (2, [(0, 1)]), + (3, [(0, 2)]), + (4, [(0, 2), (2, 3)]), + (5, [(0, 2), (2, 4)]), + (6, [(0, 2), (2, 4), (4, 5)]), + ], + ) + def test_task_segmentation(self, size, expected_segments): + task_spec = { + "name": f"test {self._USERNAME} to check segmentation into jobs", + "labels": [{"name": "car"}], + "segment_size": 3, + "overlap": 1, + } + + task_data = { + "image_quality": 75, + "client_files": generate_image_files(size), + } + + task_id, _ = create_task(self._USERNAME, task_spec, task_data) + + # check task size + with make_api_client(self._USERNAME) as api_client: + paginated_job_list, _ = api_client.jobs_api.list(task_id=task_id) + + jobs = paginated_job_list.results + jobs.sort(key=lambda job: job.start_frame) + + assert [(j.start_frame, j.stop_frame) for j in jobs] == expected_segments + def test_can_create_task_with_exif_rotated_images(self): task_spec = { "name": f"test {self._USERNAME} to create a task with exif rotated images", diff --git a/tests/python/shared/utils/helpers.py b/tests/python/shared/utils/helpers.py index 312fb99f66c..f336cb3f911 100644 --- a/tests/python/shared/utils/helpers.py +++ b/tests/python/shared/utils/helpers.py @@ -6,6 +6,8 @@ from io import BytesIO from typing import List, Optional +import av +import av.video.reformatter from PIL import Image from shared.fixtures.init import get_server_image_tag @@ -38,6 +40,26 @@ def generate_image_files( return images +def generate_video_file(num_frames: int, size=(50, 50)) -> BytesIO: + f = BytesIO() + f.name = "video.avi" + + with av.open(f, "w") as container: + stream = container.add_stream("mjpeg", rate=60) + stream.width = size[0] + stream.height = size[1] + stream.color_range = av.video.reformatter.ColorRange.JPEG + + for i in range(num_frames): + frame = av.VideoFrame.from_image(Image.new("RGB", size=size, color=(i, i, i))) + for packet in stream.encode(frame): + container.mux(packet) + + f.seek(0) + + return f + + def generate_manifest(path: str) -> None: command = [ "docker",