diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 97ac83f99d7..3623e0d66a5 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -32,7 +32,7 @@ import cvat.apps.dataset_manager as dm from cvat.apps.dataset_manager.bindings import CvatTaskOrJobDataExtractor, TaskData from cvat.apps.dataset_manager.task import TaskAnnotation -from cvat.apps.dataset_manager.views import clear_export_cache, export, parse_export_filename +from cvat.apps.dataset_manager.views import clear_export_cache, export, parse_export_file_path from cvat.apps.engine.models import Task from cvat.apps.engine.tests.utils import get_paginated_collection, ApiTestBase, ForceLogin @@ -1524,7 +1524,7 @@ def _clear(*_, file_path: str, file_ctime: str): first_export_path = export(dst_format=format_name, task_id=task_id) - export_instance_time = parse_export_filename(first_export_path).instance_timestamp + export_instance_timestamp = parse_export_file_path(first_export_path).instance_timestamp self._create_annotations(task, f'{format_name} many jobs', "default") @@ -1548,7 +1548,7 @@ def _clear(*_, file_path: str, file_ctime: str): export_checked_the_file, export_created_the_file, export_file_path, clear_removed_the_file, ), - kwargs=dict(file_path=first_export_path, file_ctime=export_instance_time), + kwargs=dict(file_path=first_export_path, file_ctime=export_instance_timestamp), ))) export_process.start() @@ -1735,7 +1735,7 @@ def patched_export(*args, **kwargs): response = self._get_request_with_data(download_url, download_params, self.admin) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - export_instance_time = parse_export_filename(export_path).instance_timestamp + export_instance_time = parse_export_file_path(export_path).instance_timestamp download_params["action"] = "download" @@ -1891,7 +1891,7 @@ def test_cleanup_can_remove_file(self): mock_rq_get_current_job.return_value = MagicMock(timeout=5) export_path = export(dst_format=format_name, task_id=task_id) - file_ctime = parse_export_filename(export_path).instance_timestamp + file_ctime = parse_export_file_path(export_path).instance_timestamp clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock()) self.assertFalse(osp.isfile(export_path)) @@ -1924,7 +1924,7 @@ def test_cleanup_can_request_retry_on_locking_failure(self): mock_rq_job = MagicMock(timeout=5) mock_rq_get_current_job.return_value = mock_rq_job - file_ctime = parse_export_filename(export_path).instance_timestamp + file_ctime = parse_export_file_path(export_path).instance_timestamp clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock()) mock_get_export_cache_lock.assert_called() @@ -1968,7 +1968,7 @@ def test_cleanup_can_defer_removal_if_file_is_used_recently(self): mock_rq_get_current_job.return_value = mock_rq_job export_path = export(dst_format=format_name, task_id=task_id) - file_ctime = parse_export_filename(export_path).instance_timestamp + file_ctime = parse_export_file_path(export_path).instance_timestamp clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock()) self.assertEqual(mock_rq_job.retries_left, 1) @@ -1994,7 +1994,7 @@ def test_cleanup_can_be_called_with_old_signature(self): export_path = export(dst_format=format_name, task_id=task_id) - file_ctime = parse_export_filename(export_path).instance_timestamp + file_ctime = parse_export_file_path(export_path).instance_timestamp old_kwargs = { 'file_path': export_path, 'file_ctime': file_ctime, diff --git a/cvat/apps/dataset_manager/util.py b/cvat/apps/dataset_manager/util.py index 1be5dd109b8..e910245a2db 100644 --- a/cvat/apps/dataset_manager/util.py +++ b/cvat/apps/dataset_manager/util.py @@ -99,7 +99,7 @@ class LockNotAvailableError(Exception): def make_export_cache_lock_key(filename: os.PathLike[str]) -> str: - return f"export_lock:{filename}" + return f"export_lock:{os.fspath(filename)}" @contextmanager @@ -160,18 +160,18 @@ def get_export_cache_dir(db_instance: Project | Task | Job) -> str: def make_export_filename( dst_dir: str, save_images: bool, - instance_update_time: datetime, + instance_timestamp: float, format_name: str, ) -> str: from .formats.registry import EXPORT_FORMATS file_ext = EXPORT_FORMATS[format_name].EXT - filename = '%s-instance%s-%s.%s' % ( + filename = '%s-instance%f-%s.%s' % ( 'dataset' if save_images else 'annotations', # store the instance timestamp in the file name to reliably get this information # ctime / mtime do not return file creation time on linux # mtime is used for file usage checks - instance_update_time.timestamp(), + instance_timestamp, make_file_name(to_snake_case(format_name)), file_ext, ) @@ -187,11 +187,11 @@ class ParsedExportFilename: file_ext: str -def parse_export_filename(file_path: os.PathLike[str]) -> ParsedExportFilename: +def parse_export_file_path(file_path: os.PathLike[str]) -> ParsedExportFilename: file_path = osp.normpath(file_path) dirname, basename = osp.split(file_path) - basename_match = re.match( + basename_match = re.fullmatch( ( r'(?Pdataset|annotations)' r'(?:-instance(?P\d+\.\d+))?' # optional for backward compatibility diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index e21ffed3595..53cbdd5c03b 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -24,7 +24,7 @@ LockNotAvailableError, current_function_name, get_export_cache_lock, get_export_cache_dir, make_export_filename, - parse_export_filename + parse_export_file_path ) from .util import EXPORT_CACHE_DIR_NAME # pylint: disable=unused-import @@ -90,7 +90,9 @@ def export(dst_format, project_id=None, task_id=None, job_id=None, server_url=No )) instance_update_time = max(tasks_update + [instance_update_time]) - output_path = make_export_filename(cache_dir, save_images, instance_update_time, dst_format) + output_path = make_export_filename( + cache_dir, save_images, instance_update_time.timestamp(), dst_format + ) os.makedirs(cache_dir, exist_ok=True) @@ -172,7 +174,7 @@ def clear_export_cache(file_path: str, file_ctime: float, logger: logging.Logger if not osp.exists(file_path): raise FileNotFoundError("Export cache file '{}' doesn't exist".format(file_path)) - parsed_filename = parse_export_filename(file_path) + parsed_filename = parse_export_file_path(file_path) cache_ttl = get_export_cache_ttl(parsed_filename.instance_type) if timezone.now().timestamp() <= osp.getmtime(file_path) + cache_ttl.total_seconds():