Skip to content

Removal of redundant tomography context code #381

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 25, 2024
Merged

Removal of redundant tomography context code #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b2487ef
Try cutting the required position files keys
stephen-riggs Oct 24, 2024
6f671b8
Bug as part 0 can be a minus sign
stephen-riggs Oct 24, 2024
b8ea668
Fairly sure there's no way to reach these bits of code
stephen-riggs Oct 24, 2024
1f66d8f
Why check tilt series which we haven't added a file for?
stephen-riggs Oct 24, 2024
1554e75
Check completeness after every tilt
stephen-riggs Oct 25, 2024
34c0ad2
Tests after _last_transferred_file removed
stephen-riggs Oct 25, 2024
aed113c
Forgot to delete this
stephen-riggs Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 12 additions & 170 deletions src/murfey/client/contexts/tomo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from datetime import datetime
from pathlib import Path
from threading import RLock
from typing import Callable, Dict, List, Optional, OrderedDict
from typing import Callable, Dict, List, OrderedDict

import requests
import xmltodict
Expand Down Expand Up @@ -38,7 +38,7 @@ def _get_tilt_angle_v5_11(p: Path) -> str:

def _find_angle_index(split_name: List[str]) -> int:
for i, part in enumerate(split_name):
if "." in part and part[0].isnumeric():
if "." in part and part[-1].isnumeric():
return i
return -1

Expand Down Expand Up @@ -110,8 +110,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
self._tilt_series_sizes: Dict[str, int] = {}
self._completed_tilt_series: List[str] = []
self._aligned_tilt_series: List[str] = []
self._motion_corrected_tilt_series: Dict[str, List[Path]] = {}
self._last_transferred_file: Path | None = None
self._data_collection_stash: list = []
self._processing_job_stash: dict = {}
self._preprocessing_triggers: dict = {}
Expand Down Expand Up @@ -156,57 +154,6 @@ def _flush_preprocess(self, tag: str, app_id: int):
capture_post(tr[0], json=process_file)
self._preprocessing_triggers.pop(tag)

def _check_for_alignment(
self,
movie_path: Path,
motion_corrected_path: Path,
url: str,
dcid: int,
pjid: int,
appid: int,
mvid: int,
tilt_angles: List,
manual_tilt_offset: Optional[float],
pixel_size: Optional[float],
):
tilt_series = _construct_tilt_series_name(movie_path)

if self._motion_corrected_tilt_series.get(
tilt_series
) and motion_corrected_path not in self._motion_corrected_tilt_series.get(
tilt_series, {}
):
self._motion_corrected_tilt_series[tilt_series].append(
motion_corrected_path
)
else:
self._motion_corrected_tilt_series[tilt_series] = [motion_corrected_path]
if tilt_series in self._completed_tilt_series:
if (
len(self._motion_corrected_tilt_series[tilt_series])
== len(self._tilt_series[tilt_series])
and len(self._motion_corrected_tilt_series[tilt_series]) > 1
and tilt_series not in self._aligned_tilt_series
):
try:

series_data: dict = {
"name": tilt_series,
"file_tilt_list": str(tilt_angles),
"dcid": dcid,
"processing_job": pjid,
"autoproc_program_id": appid,
"motion_corrected_path": str(motion_corrected_path),
"movie_id": mvid,
"manual_tilt_offset": manual_tilt_offset,
"pixel_size": pixel_size,
}
capture_post(url, json=series_data)
with self._lock:
self._aligned_tilt_series.append(tilt_series)
except Exception as e:
logger.warning(f"Data error {e}")

def _complete_process_file(
self,
incomplete_process_file: ProcessFileIncomplete,
Expand Down Expand Up @@ -302,7 +249,6 @@ def _add_tilt(
file_path: Path,
extract_tilt_angle: Callable[[Path], str],
environment: MurfeyInstanceEnvironment | None = None,
required_position_files: List[Path] | None = None,
required_strings: List[str] | None = None,
) -> List[str]:
if not environment:
Expand All @@ -312,7 +258,6 @@ def _add_tilt(
if not source:
logger.warning(f"No source found for file {file_path}")
return []
# required_position_files = required_position_files or []
required_strings = (
["fractions"] if required_strings is None else required_strings
)
Expand Down Expand Up @@ -472,24 +417,6 @@ def _add_tilt(
else:
self._tilt_series[tilt_series].append(file_path)

res = []
if self._last_transferred_file:
last_tilt_series = _construct_tilt_series_name(self._last_transferred_file)

last_tilt_angle = extract_tilt_angle(self._last_transferred_file)
self._last_transferred_file = file_path
if (
last_tilt_series != tilt_series
and last_tilt_angle != tilt_angle
or self._tilt_series_sizes.get(tilt_series)
) or self._completed_tilt_series:
res = self._check_tilt_series(
tilt_series,
required_position_files or [],
file_transferred_to,
environment=environment,
)

if environment:
tilt_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/tilt"
tilt_data = {
Expand Down Expand Up @@ -543,99 +470,28 @@ def _add_tilt(
}
capture_post(preproc_url, json=preproc_data)

self._last_transferred_file = file_path
return res
return self._check_tilt_series(tilt_series)

def _check_tilt_series(
self,
tilt_series: str,
required_position_files: List[Path],
file_transferred_to: Path | None,
environment: MurfeyInstanceEnvironment | None = None,
) -> List[str]:
newly_completed_series: List[str] = []
if not self._tilt_series:
mdoc_tilt_series_size = self._tilt_series_sizes.get(tilt_series, 0)
if not self._tilt_series or not mdoc_tilt_series_size:
return newly_completed_series
this_tilt_series_size = len(self._tilt_series.get(tilt_series, []))
tilt_series_size_check = (
(this_tilt_series_size == self._tilt_series_sizes.get(tilt_series))
if self._tilt_series_sizes.get(tilt_series)
else False
)
if tilt_series_size_check and not required_position_files:
if tilt_series not in self._completed_tilt_series:
self._completed_tilt_series.append(tilt_series)
newly_completed_series.append(tilt_series)
for ts, ta in self._tilt_series.items():
required_position_files_check = (
all(_f.is_file() for _f in required_position_files)
if required_position_files
else True
)
if self._tilt_series_sizes.get(ts):
completion_test = len(ta) >= self._tilt_series_sizes[ts]
if completion_test:
completion_test = required_position_files_check
else:
completion_test = False
if ts not in self._completed_tilt_series and completion_test:
newly_completed_series.append(ts)
self._completed_tilt_series.append(ts)
if environment and file_transferred_to:
file_tilt_list = []
movie: str
angle: str
for movie, angle in environment.tilt_angles[ts]:
if environment.motion_corrected_movies.get(Path(movie)):
file_tilt_list.append(
[
str(
environment.motion_corrected_movies[
Path(movie)
][0]
),
angle,
str(
environment.motion_corrected_movies[
Path(movie)
][1]
),
]
)
if environment.motion_corrected_movies.get(file_transferred_to):
self._check_for_alignment(
file_transferred_to,
Path(
environment.motion_corrected_movies[ # key error PosixPath
file_transferred_to
][
0
]
),
environment.url.geturl(),
environment.data_collection_ids[ts],
environment.processing_job_ids[ts]["em-tomo-align"],
environment.autoproc_program_ids[ts]["em-tomo-align"],
int(
environment.motion_corrected_movies[
file_transferred_to
][1]
),
file_tilt_list,
environment.data_collection_parameters.get(
"manual_tilt_offset"
),
environment.data_collection_parameters.get(
"pixel_size_on_image"
),
)

counted_tilts = len(self._tilt_series.get(tilt_series, []))
tilt_series_size_check = counted_tilts >= mdoc_tilt_series_size
if tilt_series_size_check and tilt_series not in self._completed_tilt_series:
self._completed_tilt_series.append(tilt_series)
newly_completed_series.append(tilt_series)
return newly_completed_series

def _add_tomo_tilt(
self,
file_path: Path,
environment: MurfeyInstanceEnvironment | None = None,
required_position_files: List[Path] | None = None,
required_strings: List[str] | None = None,
) -> List[str]:
required_strings = (
Expand All @@ -654,16 +510,10 @@ def _add_tomo_tilt(
tilt_info_extraction = tomo_tilt_info["5.7"]
else:
tilt_info_extraction = tomo_tilt_info["5.7"]
tilt_series = _construct_tilt_series_name(file_path)
return self._add_tilt(
file_path,
tilt_info_extraction,
environment=environment,
required_position_files=(
required_position_files
if required_position_files is not None
else [file_path.parent / (tilt_series + ".mdoc")]
),
required_strings=required_strings,
)

Expand Down Expand Up @@ -729,7 +579,6 @@ def post_transfer(
completed_tilts = self._add_tomo_tilt(
transferred_file,
environment=environment,
required_position_files=kwargs.get("required_position_files"),
required_strings=kwargs.get("required_strings")
or required_strings,
)
Expand All @@ -744,14 +593,7 @@ def post_transfer(
if environment:
source = self._get_source(transferred_file, environment)
if source:
completed_tilts = self._check_tilt_series(
tilt_series,
kwargs.get("required_position_files") or [],
self._file_transferred_to(
environment, source, transferred_file
),
environment=environment,
)
completed_tilts = self._check_tilt_series(tilt_series)

# Always update the tilt series length in the database after an mdoc
if environment.murfey_session is not None:
Expand Down
2 changes: 0 additions & 2 deletions src/murfey/client/instance_environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ class MurfeyInstanceEnvironment(BaseModel):
}
data_collection_parameters: dict = {}
movies: Dict[Path, MovieTracker] = {}
motion_corrected_movies: Dict[Path, List[str]] = {}
listeners: Dict[str, Set[Callable]] = {}
movie_tilt_pair: Dict[Path, str] = {}
tilt_angles: Dict[str, List[List[str]]] = {}
Expand Down Expand Up @@ -118,7 +117,6 @@ def clear(self):
self.autoproc_program_ids = {}
self.data_collection_parameters = {}
self.movies = {}
self.motion_corrected_movies = {}
self.listeners = {}
self.movie_tilt_pair = {}
self.tilt_angles = {}
Expand Down
16 changes: 2 additions & 14 deletions tests/client/test_context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

def test_tomography_context_initialisation_for_tomo(tmp_path):
context = TomographyContext("tomo", tmp_path)
assert not context._last_transferred_file
assert not context._completed_tilt_series
assert context._acquisition_software == "tomo"


Expand Down Expand Up @@ -38,10 +38,6 @@ def test_tomography_context_add_tomo_tilt(mock_post, mock_get, tmp_path):
assert context._tilt_series == {
"Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
}
assert (
context._last_transferred_file
== tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
)
(tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
context.post_transfer(
tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
Expand Down Expand Up @@ -101,10 +97,6 @@ def test_tomography_context_add_tomo_tilt_out_of_order(mock_post, mock_get, tmp_
assert context._tilt_series == {
"Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
}
assert (
context._last_transferred_file
== tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
)
(tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
context.post_transfer(
tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
Expand Down Expand Up @@ -195,10 +187,6 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_
assert context._tilt_series == {
"Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
}
assert (
context._last_transferred_file
== tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
)
(tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
context.post_transfer(
tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
Expand Down Expand Up @@ -236,7 +224,7 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_

def test_tomography_context_initialisation_for_serialem(tmp_path):
context = TomographyContext("serialem", tmp_path)
assert not context._last_transferred_file
assert not context._completed_tilt_series
assert context._acquisition_software == "serialem"


Expand Down