From b2487ef74be5feb017d03fabdb6dc84d63c6ebac Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 24 Oct 2024 14:51:04 +0100 Subject: [PATCH 1/7] Try cutting the required position files keys --- src/murfey/client/contexts/tomo.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index deb277c5a..a459aca3d 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -302,7 +302,6 @@ def _add_tilt( file_path: Path, extract_tilt_angle: Callable[[Path], str], environment: MurfeyInstanceEnvironment | None = None, - required_position_files: List[Path] | None = None, required_strings: List[str] | None = None, ) -> List[str]: if not environment: @@ -312,7 +311,6 @@ def _add_tilt( if not source: logger.warning(f"No source found for file {file_path}") return [] - # required_position_files = required_position_files or [] required_strings = ( ["fractions"] if required_strings is None else required_strings ) @@ -485,7 +483,6 @@ def _add_tilt( ) or self._completed_tilt_series: res = self._check_tilt_series( tilt_series, - required_position_files or [], file_transferred_to, environment=environment, ) @@ -549,7 +546,6 @@ def _add_tilt( def _check_tilt_series( self, tilt_series: str, - required_position_files: List[Path], file_transferred_to: Path | None, environment: MurfeyInstanceEnvironment | None = None, ) -> List[str]: @@ -562,20 +558,12 @@ def _check_tilt_series( if self._tilt_series_sizes.get(tilt_series) else False ) - if tilt_series_size_check and not required_position_files: - if tilt_series not in self._completed_tilt_series: - self._completed_tilt_series.append(tilt_series) - newly_completed_series.append(tilt_series) + if tilt_series_size_check and tilt_series not in self._completed_tilt_series: + self._completed_tilt_series.append(tilt_series) + newly_completed_series.append(tilt_series) for ts, ta in self._tilt_series.items(): - required_position_files_check = ( - all(_f.is_file() for _f in required_position_files) - if required_position_files - else True - ) if self._tilt_series_sizes.get(ts): completion_test = len(ta) >= self._tilt_series_sizes[ts] - if completion_test: - completion_test = required_position_files_check else: completion_test = False if ts not in self._completed_tilt_series and completion_test: @@ -635,7 +623,6 @@ def _add_tomo_tilt( self, file_path: Path, environment: MurfeyInstanceEnvironment | None = None, - required_position_files: List[Path] | None = None, required_strings: List[str] | None = None, ) -> List[str]: required_strings = ( @@ -654,16 +641,10 @@ def _add_tomo_tilt( tilt_info_extraction = tomo_tilt_info["5.7"] else: tilt_info_extraction = tomo_tilt_info["5.7"] - tilt_series = _construct_tilt_series_name(file_path) return self._add_tilt( file_path, tilt_info_extraction, environment=environment, - required_position_files=( - required_position_files - if required_position_files is not None - else [file_path.parent / (tilt_series + ".mdoc")] - ), required_strings=required_strings, ) @@ -729,7 +710,6 @@ def post_transfer( completed_tilts = self._add_tomo_tilt( transferred_file, environment=environment, - required_position_files=kwargs.get("required_position_files"), required_strings=kwargs.get("required_strings") or required_strings, ) @@ -746,7 +726,6 @@ def post_transfer( if source: completed_tilts = self._check_tilt_series( tilt_series, - kwargs.get("required_position_files") or [], self._file_transferred_to( environment, source, transferred_file ), From 6f671b8e1f187d12f7cb1c6f37c43f0348ef5644 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 24 Oct 2024 17:11:31 +0100 Subject: [PATCH 2/7] Bug as part 0 can be a minus sign --- src/murfey/client/contexts/tomo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index a459aca3d..85fab06cf 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -38,7 +38,7 @@ def _get_tilt_angle_v5_11(p: Path) -> str: def _find_angle_index(split_name: List[str]) -> int: for i, part in enumerate(split_name): - if "." in part and part[0].isnumeric(): + if "." in part and part[-1].isnumeric(): return i return -1 From b8ea668d861cd1e44fc23b8ee845511c3ba8d129 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 24 Oct 2024 17:15:06 +0100 Subject: [PATCH 3/7] Fairly sure there's no way to reach these bits of code --- src/murfey/client/contexts/tomo.py | 102 +--------------------- src/murfey/client/instance_environment.py | 1 - 2 files changed, 1 insertion(+), 102 deletions(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 85fab06cf..eacf502c2 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -4,7 +4,7 @@ from datetime import datetime from pathlib import Path from threading import RLock -from typing import Callable, Dict, List, Optional, OrderedDict +from typing import Callable, Dict, List, OrderedDict import requests import xmltodict @@ -110,7 +110,6 @@ def __init__(self, acquisition_software: str, basepath: Path): self._tilt_series_sizes: Dict[str, int] = {} self._completed_tilt_series: List[str] = [] self._aligned_tilt_series: List[str] = [] - self._motion_corrected_tilt_series: Dict[str, List[Path]] = {} self._last_transferred_file: Path | None = None self._data_collection_stash: list = [] self._processing_job_stash: dict = {} @@ -156,57 +155,6 @@ def _flush_preprocess(self, tag: str, app_id: int): capture_post(tr[0], json=process_file) self._preprocessing_triggers.pop(tag) - def _check_for_alignment( - self, - movie_path: Path, - motion_corrected_path: Path, - url: str, - dcid: int, - pjid: int, - appid: int, - mvid: int, - tilt_angles: List, - manual_tilt_offset: Optional[float], - pixel_size: Optional[float], - ): - tilt_series = _construct_tilt_series_name(movie_path) - - if self._motion_corrected_tilt_series.get( - tilt_series - ) and motion_corrected_path not in self._motion_corrected_tilt_series.get( - tilt_series, {} - ): - self._motion_corrected_tilt_series[tilt_series].append( - motion_corrected_path - ) - else: - self._motion_corrected_tilt_series[tilt_series] = [motion_corrected_path] - if tilt_series in self._completed_tilt_series: - if ( - len(self._motion_corrected_tilt_series[tilt_series]) - == len(self._tilt_series[tilt_series]) - and len(self._motion_corrected_tilt_series[tilt_series]) > 1 - and tilt_series not in self._aligned_tilt_series - ): - try: - - series_data: dict = { - "name": tilt_series, - "file_tilt_list": str(tilt_angles), - "dcid": dcid, - "processing_job": pjid, - "autoproc_program_id": appid, - "motion_corrected_path": str(motion_corrected_path), - "movie_id": mvid, - "manual_tilt_offset": manual_tilt_offset, - "pixel_size": pixel_size, - } - capture_post(url, json=series_data) - with self._lock: - self._aligned_tilt_series.append(tilt_series) - except Exception as e: - logger.warning(f"Data error {e}") - def _complete_process_file( self, incomplete_process_file: ProcessFileIncomplete, @@ -569,54 +517,6 @@ def _check_tilt_series( if ts not in self._completed_tilt_series and completion_test: newly_completed_series.append(ts) self._completed_tilt_series.append(ts) - if environment and file_transferred_to: - file_tilt_list = [] - movie: str - angle: str - for movie, angle in environment.tilt_angles[ts]: - if environment.motion_corrected_movies.get(Path(movie)): - file_tilt_list.append( - [ - str( - environment.motion_corrected_movies[ - Path(movie) - ][0] - ), - angle, - str( - environment.motion_corrected_movies[ - Path(movie) - ][1] - ), - ] - ) - if environment.motion_corrected_movies.get(file_transferred_to): - self._check_for_alignment( - file_transferred_to, - Path( - environment.motion_corrected_movies[ # key error PosixPath - file_transferred_to - ][ - 0 - ] - ), - environment.url.geturl(), - environment.data_collection_ids[ts], - environment.processing_job_ids[ts]["em-tomo-align"], - environment.autoproc_program_ids[ts]["em-tomo-align"], - int( - environment.motion_corrected_movies[ - file_transferred_to - ][1] - ), - file_tilt_list, - environment.data_collection_parameters.get( - "manual_tilt_offset" - ), - environment.data_collection_parameters.get( - "pixel_size_on_image" - ), - ) return newly_completed_series def _add_tomo_tilt( diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index d2f56b540..15e46cf8f 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -53,7 +53,6 @@ class MurfeyInstanceEnvironment(BaseModel): } data_collection_parameters: dict = {} movies: Dict[Path, MovieTracker] = {} - motion_corrected_movies: Dict[Path, List[str]] = {} listeners: Dict[str, Set[Callable]] = {} movie_tilt_pair: Dict[Path, str] = {} tilt_angles: Dict[str, List[List[str]]] = {} From 1f66d8f9d82d564e855376861ed442963a3b3137 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 24 Oct 2024 17:25:01 +0100 Subject: [PATCH 4/7] Why check tilt series which we haven't added a file for? --- src/murfey/client/contexts/tomo.py | 36 ++++++------------------------ 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index eacf502c2..4fee8fd5b 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -429,11 +429,7 @@ def _add_tilt( and last_tilt_angle != tilt_angle or self._tilt_series_sizes.get(tilt_series) ) or self._completed_tilt_series: - res = self._check_tilt_series( - tilt_series, - file_transferred_to, - environment=environment, - ) + res = self._check_tilt_series(tilt_series) if environment: tilt_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/tilt" @@ -494,29 +490,17 @@ def _add_tilt( def _check_tilt_series( self, tilt_series: str, - file_transferred_to: Path | None, - environment: MurfeyInstanceEnvironment | None = None, ) -> List[str]: newly_completed_series: List[str] = [] - if not self._tilt_series: + mdoc_tilt_series_size = self._tilt_series_sizes.get(tilt_series, 0) + if not self._tilt_series or not mdoc_tilt_series_size: return newly_completed_series - this_tilt_series_size = len(self._tilt_series.get(tilt_series, [])) - tilt_series_size_check = ( - (this_tilt_series_size == self._tilt_series_sizes.get(tilt_series)) - if self._tilt_series_sizes.get(tilt_series) - else False - ) + + counted_tilts = len(self._tilt_series.get(tilt_series, [])) + tilt_series_size_check = counted_tilts >= mdoc_tilt_series_size if tilt_series_size_check and tilt_series not in self._completed_tilt_series: self._completed_tilt_series.append(tilt_series) newly_completed_series.append(tilt_series) - for ts, ta in self._tilt_series.items(): - if self._tilt_series_sizes.get(ts): - completion_test = len(ta) >= self._tilt_series_sizes[ts] - else: - completion_test = False - if ts not in self._completed_tilt_series and completion_test: - newly_completed_series.append(ts) - self._completed_tilt_series.append(ts) return newly_completed_series def _add_tomo_tilt( @@ -624,13 +608,7 @@ def post_transfer( if environment: source = self._get_source(transferred_file, environment) if source: - completed_tilts = self._check_tilt_series( - tilt_series, - self._file_transferred_to( - environment, source, transferred_file - ), - environment=environment, - ) + completed_tilts = self._check_tilt_series(tilt_series) # Always update the tilt series length in the database after an mdoc if environment.murfey_session is not None: From 1554e7554cb48dc07bb2a0ef47e48d66ab53de05 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Oct 2024 10:11:43 +0100 Subject: [PATCH 5/7] Check completeness after every tilt --- src/murfey/client/contexts/tomo.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 4fee8fd5b..6080a7bc0 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -110,7 +110,6 @@ def __init__(self, acquisition_software: str, basepath: Path): self._tilt_series_sizes: Dict[str, int] = {} self._completed_tilt_series: List[str] = [] self._aligned_tilt_series: List[str] = [] - self._last_transferred_file: Path | None = None self._data_collection_stash: list = [] self._processing_job_stash: dict = {} self._preprocessing_triggers: dict = {} @@ -418,19 +417,6 @@ def _add_tilt( else: self._tilt_series[tilt_series].append(file_path) - res = [] - if self._last_transferred_file: - last_tilt_series = _construct_tilt_series_name(self._last_transferred_file) - - last_tilt_angle = extract_tilt_angle(self._last_transferred_file) - self._last_transferred_file = file_path - if ( - last_tilt_series != tilt_series - and last_tilt_angle != tilt_angle - or self._tilt_series_sizes.get(tilt_series) - ) or self._completed_tilt_series: - res = self._check_tilt_series(tilt_series) - if environment: tilt_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/tilt" tilt_data = { @@ -484,8 +470,7 @@ def _add_tilt( } capture_post(preproc_url, json=preproc_data) - self._last_transferred_file = file_path - return res + return self._check_tilt_series(tilt_series) def _check_tilt_series( self, From 34c0ad2e96fe480e900cdaf390f291b789582e04 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Oct 2024 10:17:28 +0100 Subject: [PATCH 6/7] Tests after _last_transferred_file removed --- tests/client/test_context.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/client/test_context.py b/tests/client/test_context.py index f4a1eaedf..d9cf4bf2b 100644 --- a/tests/client/test_context.py +++ b/tests/client/test_context.py @@ -10,7 +10,7 @@ def test_tomography_context_initialisation_for_tomo(tmp_path): context = TomographyContext("tomo", tmp_path) - assert not context._last_transferred_file + assert not context._completed_tilt_series assert context._acquisition_software == "tomo" @@ -38,10 +38,6 @@ def test_tomography_context_add_tomo_tilt(mock_post, mock_get, tmp_path): assert context._tilt_series == { "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"] } - assert ( - context._last_transferred_file - == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff" - ) (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch() context.post_transfer( tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff", @@ -101,10 +97,6 @@ def test_tomography_context_add_tomo_tilt_out_of_order(mock_post, mock_get, tmp_ assert context._tilt_series == { "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"] } - assert ( - context._last_transferred_file - == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff" - ) (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch() context.post_transfer( tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff", @@ -195,10 +187,6 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_ assert context._tilt_series == { "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"] } - assert ( - context._last_transferred_file - == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff" - ) (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch() context.post_transfer( tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff", @@ -236,7 +224,7 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_ def test_tomography_context_initialisation_for_serialem(tmp_path): context = TomographyContext("serialem", tmp_path) - assert not context._last_transferred_file + assert not context._completed_tilt_series assert context._acquisition_software == "serialem" From aed113cc9421555d7e05222382fbeb3812721198 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Oct 2024 10:27:31 +0100 Subject: [PATCH 7/7] Forgot to delete this --- src/murfey/client/instance_environment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 15e46cf8f..95af5074c 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -117,7 +117,6 @@ def clear(self): self.autoproc_program_ids = {} self.data_collection_parameters = {} self.movies = {} - self.motion_corrected_movies = {} self.listeners = {} self.movie_tilt_pair = {} self.tilt_angles = {}