Skip to content

Commit

Permalink
Fix comments
Browse files Browse the repository at this point in the history
  • Loading branch information
zhiltsov-max committed May 27, 2024
1 parent e76382d commit 2af76a8
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 17 deletions.
16 changes: 8 additions & 8 deletions cvat/apps/dataset_manager/tests/test_rest_api_formats.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import cvat.apps.dataset_manager as dm
from cvat.apps.dataset_manager.bindings import CvatTaskOrJobDataExtractor, TaskData
from cvat.apps.dataset_manager.task import TaskAnnotation
from cvat.apps.dataset_manager.views import clear_export_cache, export, parse_export_filename
from cvat.apps.dataset_manager.views import clear_export_cache, export, parse_export_file_path
from cvat.apps.engine.models import Task
from cvat.apps.engine.tests.utils import get_paginated_collection, ApiTestBase, ForceLogin

Expand Down Expand Up @@ -1524,7 +1524,7 @@ def _clear(*_, file_path: str, file_ctime: str):

first_export_path = export(dst_format=format_name, task_id=task_id)

export_instance_time = parse_export_filename(first_export_path).instance_timestamp
export_instance_timestamp = parse_export_file_path(first_export_path).instance_timestamp

self._create_annotations(task, f'{format_name} many jobs', "default")

Expand All @@ -1548,7 +1548,7 @@ def _clear(*_, file_path: str, file_ctime: str):
export_checked_the_file, export_created_the_file,
export_file_path, clear_removed_the_file,
),
kwargs=dict(file_path=first_export_path, file_ctime=export_instance_time),
kwargs=dict(file_path=first_export_path, file_ctime=export_instance_timestamp),
)))

export_process.start()
Expand Down Expand Up @@ -1735,7 +1735,7 @@ def patched_export(*args, **kwargs):
response = self._get_request_with_data(download_url, download_params, self.admin)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)

export_instance_time = parse_export_filename(export_path).instance_timestamp
export_instance_time = parse_export_file_path(export_path).instance_timestamp

download_params["action"] = "download"

Expand Down Expand Up @@ -1891,7 +1891,7 @@ def test_cleanup_can_remove_file(self):
mock_rq_get_current_job.return_value = MagicMock(timeout=5)

export_path = export(dst_format=format_name, task_id=task_id)
file_ctime = parse_export_filename(export_path).instance_timestamp
file_ctime = parse_export_file_path(export_path).instance_timestamp
clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock())

self.assertFalse(osp.isfile(export_path))
Expand Down Expand Up @@ -1924,7 +1924,7 @@ def test_cleanup_can_request_retry_on_locking_failure(self):
mock_rq_job = MagicMock(timeout=5)
mock_rq_get_current_job.return_value = mock_rq_job

file_ctime = parse_export_filename(export_path).instance_timestamp
file_ctime = parse_export_file_path(export_path).instance_timestamp
clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock())

mock_get_export_cache_lock.assert_called()
Expand Down Expand Up @@ -1968,7 +1968,7 @@ def test_cleanup_can_defer_removal_if_file_is_used_recently(self):
mock_rq_get_current_job.return_value = mock_rq_job

export_path = export(dst_format=format_name, task_id=task_id)
file_ctime = parse_export_filename(export_path).instance_timestamp
file_ctime = parse_export_file_path(export_path).instance_timestamp
clear_export_cache(file_path=export_path, file_ctime=file_ctime, logger=MagicMock())

self.assertEqual(mock_rq_job.retries_left, 1)
Expand All @@ -1994,7 +1994,7 @@ def test_cleanup_can_be_called_with_old_signature(self):

export_path = export(dst_format=format_name, task_id=task_id)

file_ctime = parse_export_filename(export_path).instance_timestamp
file_ctime = parse_export_file_path(export_path).instance_timestamp
old_kwargs = {
'file_path': export_path,
'file_ctime': file_ctime,
Expand Down
12 changes: 6 additions & 6 deletions cvat/apps/dataset_manager/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ class LockNotAvailableError(Exception):


def make_export_cache_lock_key(filename: os.PathLike[str]) -> str:
return f"export_lock:{filename}"
return f"export_lock:{os.fspath(filename)}"


@contextmanager
Expand Down Expand Up @@ -160,18 +160,18 @@ def get_export_cache_dir(db_instance: Project | Task | Job) -> str:
def make_export_filename(
dst_dir: str,
save_images: bool,
instance_update_time: datetime,
instance_timestamp: float,
format_name: str,
) -> str:
from .formats.registry import EXPORT_FORMATS
file_ext = EXPORT_FORMATS[format_name].EXT

filename = '%s-instance%s-%s.%s' % (
filename = '%s-instance%f-%s.%s' % (
'dataset' if save_images else 'annotations',
# store the instance timestamp in the file name to reliably get this information
# ctime / mtime do not return file creation time on linux
# mtime is used for file usage checks
instance_update_time.timestamp(),
instance_timestamp,
make_file_name(to_snake_case(format_name)),
file_ext,
)
Expand All @@ -187,11 +187,11 @@ class ParsedExportFilename:
file_ext: str


def parse_export_filename(file_path: os.PathLike[str]) -> ParsedExportFilename:
def parse_export_file_path(file_path: os.PathLike[str]) -> ParsedExportFilename:
file_path = osp.normpath(file_path)
dirname, basename = osp.split(file_path)

basename_match = re.match(
basename_match = re.fullmatch(
(
r'(?P<export_mode>dataset|annotations)'
r'(?:-instance(?P<instance_timestamp>\d+\.\d+))?' # optional for backward compatibility
Expand Down
8 changes: 5 additions & 3 deletions cvat/apps/dataset_manager/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
LockNotAvailableError,
current_function_name, get_export_cache_lock,
get_export_cache_dir, make_export_filename,
parse_export_filename
parse_export_file_path
)
from .util import EXPORT_CACHE_DIR_NAME # pylint: disable=unused-import

Expand Down Expand Up @@ -90,7 +90,9 @@ def export(dst_format, project_id=None, task_id=None, job_id=None, server_url=No
))
instance_update_time = max(tasks_update + [instance_update_time])

output_path = make_export_filename(cache_dir, save_images, instance_update_time, dst_format)
output_path = make_export_filename(
cache_dir, save_images, instance_update_time.timestamp(), dst_format
)

os.makedirs(cache_dir, exist_ok=True)

Expand Down Expand Up @@ -172,7 +174,7 @@ def clear_export_cache(file_path: str, file_ctime: float, logger: logging.Logger
if not osp.exists(file_path):
raise FileNotFoundError("Export cache file '{}' doesn't exist".format(file_path))

parsed_filename = parse_export_filename(file_path)
parsed_filename = parse_export_file_path(file_path)
cache_ttl = get_export_cache_ttl(parsed_filename.instance_type)

if timezone.now().timestamp() <= osp.getmtime(file_path) + cache_ttl.total_seconds():
Expand Down

0 comments on commit 2af76a8

Please sign in to comment.