From 10b61cab085f46434b33e9dd19519198ec6328fd Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 24 Jun 2025 16:26:45 +0100 Subject: [PATCH 01/32] Add a way of reading tomo metadata files --- src/murfey/client/contexts/tomo_metadata.py | 316 ++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 src/murfey/client/contexts/tomo_metadata.py diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py new file mode 100644 index 000000000..4b81d11a1 --- /dev/null +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -0,0 +1,316 @@ +import logging +from pathlib import Path +from typing import Optional + +import requests +import xmltodict + +from murfey.client.context import Context +from murfey.client.contexts.spa import _file_transferred_to, _get_source +from murfey.client.contexts.spa_metadata import _atlas_destination +from murfey.client.instance_environment import MurfeyInstanceEnvironment, SampleInfo +from murfey.util.api import url_path_for +from murfey.util.client import authorised_requests, capture_post + +logger = logging.getLogger("murfey.client.contexts.tomo_metadata") + +requests.get, requests.post, requests.put, requests.delete = authorised_requests() + + +def get_visitless_source( + transferred_file: Path, environment: MurfeyInstanceEnvironment +) -> Optional[str]: + source = _get_source(transferred_file, environment=environment) + visitless_source_search_dir = str(source).replace(f"/{environment.visit}", "") + visitless_source_images_dirs = sorted( + Path(visitless_source_search_dir).glob("Images-Disc*"), + key=lambda x: x.stat().st_ctime, + ) + if not visitless_source_images_dirs: + logger.warning(f"Cannot find Images-Disc* in {visitless_source_search_dir}") + return None + visitless_source = str(visitless_source_images_dirs[-1]) + return visitless_source + + +class TomographyMetadataContext(Context): + def __init__(self, acquisition_software: str, basepath: Path): + super().__init__("Tomography_metadata", acquisition_software) + self._basepath = basepath + + def post_transfer( + self, + transferred_file: Path, + environment: Optional[MurfeyInstanceEnvironment] = None, + **kwargs, + ): + super().post_transfer( + transferred_file=transferred_file, + environment=environment, + **kwargs, + ) + + if transferred_file.name == "Session.xml" and environment: + logger.info("Tomography session metadata found") + with open(transferred_file, "r") as session_xml: + session_data = xmltodict.parse(session_xml.read()) + + windows_path = session_data["TomographySession"]["AtlasId"]["#text"] + logger.info(f"Windows path to atlas metadata found: {windows_path}") + visit_index = windows_path.split("\\").index(environment.visit) + partial_path = "/".join(windows_path.split("\\")[visit_index + 1 :]) + logger.info("Partial Linux path successfully constructed from Windows path") + + source = _get_source(transferred_file, environment) + if not source: + logger.warning( + f"Source could not be identified for {str(transferred_file)}" + ) + return + + source_visit_dir = source.parent + + logger.info( + f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}" + ) + atlas_xml_path = list( + (source_visit_dir / partial_path).parent.glob("Atlas_*.xml") + )[0] + logger.info(f"Atlas XML path {str(atlas_xml_path)} found") + with open(atlas_xml_path, "rb") as atlas_xml: + atlas_xml_data = xmltodict.parse(atlas_xml) + atlas_pixel_size = float( + atlas_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][ + "numericValue" + ] + ) + atlas_binning = int( + atlas_xml_data["MicroscopeImage"]["microscopeData"]["acquisition"][ + "camera" + ]["Binning"]["a:x"] + ) + + for p in partial_path.split("/"): + if p.startswith("Sample"): + sample = int(p.replace("Sample", "")) + break + else: + logger.warning(f"Sample could not be identified for {transferred_file}") + return + if source: + environment.samples[source] = SampleInfo( + atlas=Path(partial_path), sample=sample + ) + url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}" + dcg_search_dir = "/".join( + p for p in transferred_file.parent.parts if p != environment.visit + ) + dcg_search_dir = ( + dcg_search_dir[1:] + if dcg_search_dir.startswith("//") + else dcg_search_dir + ) + dcg_images_dirs = sorted( + Path(dcg_search_dir).glob("Images-Disc*"), + key=lambda x: x.stat().st_ctime, + ) + if not dcg_images_dirs: + logger.warning(f"Cannot find Images-Disc* in {dcg_search_dir}") + return + dcg_tag = str(dcg_images_dirs[-1]) + dcg_data = { + "experiment_type": "tomo", + "experiment_type_id": 36, + "tag": dcg_tag, + "atlas": str( + _atlas_destination(environment, source, transferred_file) + / environment.samples[source].atlas.parent + / atlas_xml_path.with_suffix(".jpg").name + ), + "sample": environment.samples[source].sample, + "atlas_pixel_size": atlas_pixel_size, + "atlas_binning": atlas_binning, + } + capture_post(url, json=dcg_data) + + elif transferred_file.name == "SearchMap.xml" and environment: + logger.info("Tomography session search map xml found") + with open(transferred_file, "r") as sm_xml: + sm_data = xmltodict.parse(sm_xml.read()) + + # This bit gets SearchMap location on Atlas + sm_pixel_size = float( + sm_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][ + "numericValue" + ] + ) + stage_position = sm_data["MicroscopeImage"]["microscopeData"]["stage"][ + "Position" + ] + sm_binning = float( + sm_data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][ + "Binning" + ]["a:x"] + ) + readout_area = sm_data["MicroscopeImage"]["microscopeData"]["acquisition"][ + "camera" + ]["ReadoutArea"] + + # Get the stage transformation + sm_transformations = sm_data["MicroscopeImage"]["CustomData"][ + "a:KeyValueOfstringanyType" + ] + stage_matrix: dict[str, float] = {} + image_matrix: dict[str, float] = {} + for key_val in sm_transformations: + if key_val["a:Key"] == "ReferenceCorrectionForStage": + stage_matrix = { + "m11": float(key_val["a:Value"]["b:_m11"]), + "m12": float(key_val["a:Value"]["b:_m12"]), + "m21": float(key_val["a:Value"]["b:_m21"]), + "m22": float(key_val["a:Value"]["b:_m22"]), + } + elif key_val["a:Key"] == "ReferenceCorrectionForImageShift": + image_matrix = { + "m11": float(key_val["a:Value"]["b:_m11"]), + "m12": float(key_val["a:Value"]["b:_m12"]), + "m21": float(key_val["a:Value"]["b:_m21"]), + "m22": float(key_val["a:Value"]["b:_m22"]), + } + if not stage_matrix or not image_matrix: + print("No matrix found") + + ref_matrix = { + "m11": float( + sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][ + "a:_m11" + ] + ), + "m12": float( + sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][ + "a:_m12" + ] + ), + "m21": float( + sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][ + "a:_m21" + ] + ), + "m22": float( + sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][ + "a:_m22" + ] + ), + } + + visitless_source = get_visitless_source(transferred_file, environment) + if not visitless_source: + return + + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" + source = _get_source(transferred_file, environment=environment) + image_path = ( + _file_transferred_to( + environment, source, transferred_file.parent / "SearchMap.jpg" + ) + if source + else "" + ) + capture_post( + sm_url, + json={ + "tag": visitless_source, + "x_stage_position": float(stage_position["X"]), + "y_stage_position": float(stage_position["Y"]), + "readout_area_x": readout_area[0], + "readout_area_y": readout_area[1], + "thumbnail_size_x": int( + (512 / max(readout_area)) * readout_area[0] + ), + "thumbnail_size_y": int( + (512 / max(readout_area)) * readout_area[1] + ), + "pixel_size": sm_pixel_size, + "image": str(image_path), + "binning": sm_binning, + "reference_matrix": ref_matrix, + "stage_correction": stage_matrix, + "image_shift_correction": image_matrix, + }, + ) + + elif transferred_file.name == "SearchMap.dm" and environment: + logger.info("Tomography session search map dm found") + with open(transferred_file, "r") as sm_xml: + sm_data = xmltodict.parse(sm_xml.read()) + + visitless_source = get_visitless_source(transferred_file, environment) + if not visitless_source: + return + + # This bit gets SearchMap location on Atlas + sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) + sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) + + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" + capture_post( + sm_url, + json={ + "tag": visitless_source, + "height": sm_height, + "width": sm_width, + }, + ) + + elif transferred_file.name == "BatchPositionsList.xml" and environment: + with open(transferred_file) as xml: + for_parsing = xml.read() + batch_xml = xmltodict.parse(for_parsing) + visitless_source = get_visitless_source(transferred_file, environment) + if not visitless_source: + return + + for batch_position in batch_xml["BatchPositionsList"]["BatchPositions"][ + "BatchPositionParameters" + ]: + batch_name = batch_position["Name"] + search_map_name = batch_position["PositionOnTileSet"]["TileSetName"] + batch_stage_location_x = float( + batch_position["PositionOnTileSet"]["StagePositionX"] + ) + batch_stage_location_y = float( + batch_position["PositionOnTileSet"]["StagePositionY"] + ) + bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=batch_name)}" + capture_post( + bp_url, + json={ + "tag": visitless_source, + "stage_position_x": batch_stage_location_x, + "stage_position_y": batch_stage_location_y, + "search_map": search_map_name, + }, + ) + + # Beamshifts + if batch_position["AdditionalExposureTemplateAreas"]: + beamshifts = batch_position["AdditionalExposureTemplateAreas"][ + "ExposureTemplateAreaParameters" + ] + if type(beamshifts) is dict: + beamshifts = [beamshifts] + for beamshift in beamshifts: + beamshift_name = beamshift["Name"] + beamshift_position_x = float(beamshift["PositionX"]) + beamshift_position_y = float(beamshift["PositionY"]) + + bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=beamshift_name)}" + capture_post( + bp_url, + json={ + "tag": visitless_source, + "stage_position_x": beamshift_position_x, + "stage_position_y": beamshift_position_y, + "search_map": search_map_name, + }, + ) From 8acd7f36e9013d8f043af6d894e5edaf3d089877 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Jun 2025 12:06:10 +0100 Subject: [PATCH 02/32] Initial code to insert searchmaps into murfey and ispyb databases --- src/murfey/client/contexts/tomo_metadata.py | 21 +-- src/murfey/server/api/session_control.py | 33 ++++ src/murfey/server/api/workflow.py | 8 +- src/murfey/server/ispyb.py | 90 ++++++++++- src/murfey/util/db.py | 19 +++ src/murfey/util/models.py | 25 +++ src/murfey/util/tomo_metadata.py | 169 ++++++++++++++++++++ 7 files changed, 347 insertions(+), 18 deletions(-) create mode 100644 src/murfey/util/tomo_metadata.py diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 4b81d11a1..e087d6ab8 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -152,9 +152,6 @@ def post_transfer( "Binning" ]["a:x"] ) - readout_area = sm_data["MicroscopeImage"]["microscopeData"]["acquisition"][ - "camera" - ]["ReadoutArea"] # Get the stage transformation sm_transformations = sm_data["MicroscopeImage"]["CustomData"][ @@ -222,14 +219,6 @@ def post_transfer( "tag": visitless_source, "x_stage_position": float(stage_position["X"]), "y_stage_position": float(stage_position["Y"]), - "readout_area_x": readout_area[0], - "readout_area_y": readout_area[1], - "thumbnail_size_x": int( - (512 / max(readout_area)) * readout_area[0] - ), - "thumbnail_size_y": int( - (512 / max(readout_area)) * readout_area[1] - ), "pixel_size": sm_pixel_size, "image": str(image_path), "binning": sm_binning, @@ -248,7 +237,7 @@ def post_transfer( if not visitless_source: return - # This bit gets SearchMap location on Atlas + # This bit gets SearchMap size sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) @@ -286,8 +275,8 @@ def post_transfer( bp_url, json={ "tag": visitless_source, - "stage_position_x": batch_stage_location_x, - "stage_position_y": batch_stage_location_y, + "x_stage_position": batch_stage_location_x, + "y_stage_position": batch_stage_location_y, "search_map": search_map_name, }, ) @@ -309,8 +298,8 @@ def post_transfer( bp_url, json={ "tag": visitless_source, - "stage_position_x": beamshift_position_x, - "stage_position_y": beamshift_position_y, + "x_stage_position": beamshift_position_x, + "y_stage_position": beamshift_position_y, "search_map": search_map_name, }, ) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 8be560b40..8ef0dc87e 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -46,12 +46,18 @@ Session, ) from murfey.util.models import ( + BatchPositionParameters, ClientInfo, FoilHoleParameters, GridSquareParameters, RsyncerInfo, + SearchMapParameters, Visit, ) +from murfey.util.tomo_metadata import ( + register_batch_position_in_database, + register_search_map_in_database, +) from murfey.workflows.spa.flush_spa_preprocess import ( register_foil_hole as _register_foil_hole, ) @@ -364,6 +370,33 @@ def register_foil_hole( return _register_foil_hole(session_id, gs_name, foil_hole_params, db) +tomography_router = APIRouter( + prefix="/session_control/tomography", + dependencies=[Depends(validate_instrument_token)], + tags=["Session Control: Tomography"], +) + + +@tomography_router.post("/sessions/{session_id}/search_map/{sm_name}") +def register_search_map( + session_id: MurfeySessionID, + sm_name: str, + search_map_params: SearchMapParameters, + db=murfey_db, +): + return register_search_map_in_database(session_id, sm_name, search_map_params, db) + + +@tomography_router.post("/sessions/{session_id}/batch_position/{batch_name}") +def register_batch_position( + session_id: MurfeySessionID, + batch_name: str, + batch_params: BatchPositionParameters, + db=murfey_db, +): + return register_batch_position_in_database(session_id, batch_name, batch_params, db) + + correlative_router = APIRouter( prefix="/session_control/correlative", dependencies=[Depends(validate_instrument_token)], diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index fe2cc71b9..d40b2ac5a 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -81,7 +81,10 @@ class DCGroupParameters(BaseModel): tag: str atlas: str = "" sample: Optional[int] = None - atlas_pixel_size: int = 0 + atlas_pixel_size: float = 0 + atlas_size_x: int = 0 + atlas_size_y: int = 0 + atlas_binning: int = 1 @router.post("/visits/{visit_name}/{session_id}/register_data_collection_group") @@ -116,6 +119,9 @@ def register_dc_group( "atlas": dcg_params.atlas, "sample": dcg_params.sample, "atlas_pixel_size": dcg_params.atlas_pixel_size, + "atlas_size_x": dcg_params.atlas_size_x, + "atlas_size_y": dcg_params.atlas_size_y, + "atlas_binning": dcg_params.atlas_binning, "dcgid": dcg_murfey[0].id, "session_id": session_id, }, diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 24f8094ce..439ce7795 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -32,7 +32,11 @@ from murfey.util import sanitise from murfey.util.config import get_security_config -from murfey.util.models import FoilHoleParameters, GridSquareParameters +from murfey.util.models import ( + FoilHoleParameters, + GridSquareParameters, + SearchMapParameters, +) log = logging.getLogger("murfey.server.ispyb") security_config = get_security_config() @@ -388,6 +392,90 @@ def do_update_foil_hole( ) return {"success": False, "return_value": None} + def do_insert_search_map( + self, + atlas_id: int, + search_map_parameters: SearchMapParameters, + ): + if ( + search_map_parameters.pixel_size + and search_map_parameters.height + and search_map_parameters.height_on_atlas + ): + search_map_parameters.pixel_size *= ( + search_map_parameters.height / search_map_parameters.height_on_atlas + ) + record = GridSquare( + atlasId=atlas_id, + gridSquareImage=search_map_parameters.image, + pixelLocationX=search_map_parameters.x_location, + pixelLocationY=search_map_parameters.y_location, + height=search_map_parameters.height_on_atlas, + width=search_map_parameters.width_on_atlas, + stageLocationX=search_map_parameters.x_stage_position, + stageLocationY=search_map_parameters.y_stage_position, + pixelSize=search_map_parameters.pixel_size, + ) + try: + with ISPyBSession() as db: + db.add(record) + db.commit() + log.info(f"Created SearchMap (GridSquare) {record.gridSquareId}") + return {"success": True, "return_value": record.gridSquareId} + except ispyb.ISPyBException as e: + log.error( + "Inserting SearchMap (GridSquare) entry caused exception '%s'.", + e, + exc_info=True, + ) + return {"success": False, "return_value": None} + + def do_update_search_map( + self, search_map_id, search_map_parameters: SearchMapParameters + ): + try: + with ISPyBSession() as db: + grid_square = ( + db.query(GridSquare) + .filter(GridSquare.gridSquareId == search_map_id) + .one() + ) + if ( + search_map_parameters.pixel_size + and search_map_parameters.height + and search_map_parameters.height_on_atlas + ): + search_map_parameters.pixel_size *= ( + search_map_parameters.height + / search_map_parameters.height_on_atlas + ) + if search_map_parameters.image: + grid_square.gridSquareImage = search_map_parameters.image + if search_map_parameters.x_location: + grid_square.pixelLocationX = search_map_parameters.x_location + if search_map_parameters.y_location: + grid_square.pixelLocationY = search_map_parameters.y_location + if search_map_parameters.height_on_atlas: + grid_square.height = search_map_parameters.height_on_atlas + if search_map_parameters.width_on_atlas: + grid_square.width = search_map_parameters.width_on_atlas + if search_map_parameters.x_stage_position: + grid_square.stageLocationX = search_map_parameters.x_stage_position + if search_map_parameters.y_stage_position: + grid_square.stageLocationY = search_map_parameters.y_stage_position + if search_map_parameters.pixel_size: + grid_square.pixelSize = search_map_parameters.pixel_size + db.add(grid_square) + db.commit() + return {"success": True, "return_value": grid_square.gridSquareId} + except ispyb.ISPyBException as e: + log.error( + "Updating SearchMap (GridSquare) entry caused exception '%s'.", + e, + exc_info=True, + ) + return {"success": False, "return_value": None} + def send(self, queue: str, message: dict, new_connection: bool = False): if self.transport: if not self.transport.is_connected(): diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8d43b98c6..d2161f168 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -372,6 +372,7 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore atlas_id: Optional[int] = None atlas_pixel_size: Optional[float] = None atlas: str = "" + atlas_binning: Optional[int] = None sample: Optional[int] = None session: Optional[Session] = Relationship(back_populates="data_collection_groups") data_collections: List["DataCollection"] = Relationship( @@ -601,6 +602,24 @@ class FoilHole(SQLModel, table=True): # type: ignore ) +class SearchMap(SQLModel, table=True): # type: ignore + id: Optional[int] = Field(primary_key=True, default=None) + session_id: int = Field(foreign_key="session.id") + name: str + tag: str + x_location: Optional[float] = None + y_location: Optional[float] = None + x_stage_position: Optional[float] = None + y_stage_position: Optional[float] = None + pixel_size: Optional[float] = None + image: str = "" + binning: Optional[float] = None + reference_matrix: Optional[dict[str, float]] = None + stage_correction: Optional[dict[str, float]] = None + image_shift_correction: Optional[dict[str, float]] = None + session: Optional[Session] = Relationship(back_populates="grid_squares") + + class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 9c80204d3..c299f1c75 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -136,6 +136,31 @@ class FoilHoleParameters(BaseModel): diameter: Optional[float] = None +class SearchMapParameters: + tag: str + x_location: Optional[float] = None + y_location: Optional[float] = None + x_stage_position: Optional[float] = None + y_stage_position: Optional[float] = None + pixel_size: Optional[float] = None + image: Optional[str] = None + binning: Optional[float] = None + reference_matrix: Optional[dict[str, float]] = None + stage_correction: Optional[dict[str, float]] = None + image_shift_correction: Optional[dict[str, float]] = None + height: Optional[int] = None + width: Optional[int] = None + height_on_atlas: Optional[int] = None + width_on_atlas: Optional[int] = None + + +class BatchPositionParameters: + tag: str + x_stage_position: float + Y_stage_position: float + search_map: str + + class MultigridWatcherSetup(BaseModel): source: Path skip_existing_processing: bool = False diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py new file mode 100644 index 000000000..3ab8ca663 --- /dev/null +++ b/src/murfey/util/tomo_metadata.py @@ -0,0 +1,169 @@ +import numpy as np +from sqlmodel import Session, select + +from murfey.server import _transport_object +from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID +from murfey.util.config import get_machine_config +from murfey.util.db import DataCollectionGroup, SearchMap +from murfey.util.db import Session as MurfeySession +from murfey.util.models import BatchPositionParameters, SearchMapParameters + + +def register_search_map_in_database( + session_id: MurfeySessionID, + search_map_name: str, + search_map_params: SearchMapParameters, + murfey_db: Session, +): + dcg = murfey_db.exec( + select(DataCollectionGroup) + .where(DataCollectionGroup.session_id == session_id) + .where(DataCollectionGroup.tag == search_map_params.tag) + ).one() + try: + search_map = murfey_db.exec( + select(SearchMap) + .where(SearchMap.name == search_map_name) + .where(SearchMap.tag == search_map_params.tag) + .where(SearchMap.session_id == session_id) + ).one() + search_map.x_stage_position = ( + search_map_params.x_stage_position or search_map.x_stage_position + ) + search_map.y_stage_position = ( + search_map_params.y_stage_position or search_map.y_stage_position + ) + search_map.pixel_size = search_map_params.pixel_size or search_map.pixel_size + search_map.image = search_map_params.image or search_map.image + search_map.binning = search_map_params.binning or search_map.binning + search_map.reference_matrix = ( + search_map_params.reference_matrix or search_map.reference_matrix + ) + search_map.stage_correction = ( + search_map_params.stage_correction or search_map.stage_correction + ) + search_map.image_shift_correction = ( + search_map_params.image_shift_correction + or search_map.image_shift_correction + ) + search_map.height = search_map_params.height or search_map.height + search_map.width = search_map_params.width or search_map.width + if _transport_object: + _transport_object.do_update_search_map(search_map.id, search_map_params) + except Exception: + if _transport_object: + sm_ispyb_response = _transport_object.do_insert_search_map( + dcg.atlas_id, search_map_params + ) + else: + # mock up response so that below still works + sm_ispyb_response = {"success": False, "return_value": None} + search_map = SearchMap( + id=( + sm_ispyb_response["return_value"] + if sm_ispyb_response["success"] + else None + ), + name=search_map_name, + session_id=session_id, + tag=search_map_params.tag, + x_stage_position=search_map_params.x_stage_position, + y_stage_position=search_map_params.y_stage_position, + pixel_size=search_map_params.pixel_size, + image=search_map_params.image, + binning=search_map_params.binning, + reference_matrix=search_map_params.reference_matrix, + stage_correction=search_map_params.stage_correction, + image_shift_correction=search_map_params.image_shift_correction, + height=search_map_params.height, + width=search_map_params.width, + ) + murfey_db.add(search_map) + murfey_db.commit() + + murfey_session = murfey_db.exec( + select(MurfeySession).where(MurfeySession.id == session_id) + ).one() + machine_config = get_machine_config(instrument_name=murfey_session.instrument_name)[ + murfey_session.instrument_name + ] + + if all( + [ + search_map.reference_matrix, + search_map.stage_correction, + search_map.x_stage_position, + search_map.y_stage_position, + search_map.pixel_size, + search_map.height, + search_map.width, + dcg.atlas_pixel_size, + dcg.atlas_binning, + ] + ): + M = np.array( + [ + [ + search_map.reference_matrix["m11"], + search_map.reference_matrix["m12"], + ], + [ + search_map.reference_matrix["m21"], + search_map.reference_matrix["m22"], + ], + ] + ) + B = np.array([search_map.x_stage_position, search_map.y_stage_position]) + R = np.array( + [ + [ + search_map.stage_correction["m11"], + search_map.stage_correction["m12"], + ], + [ + search_map.stage_correction["m21"], + search_map.stage_correction["m22"], + ], + ] + ) + M_corrected = np.matmul(np.linalg.inv(M), R) + vector_pixel = np.matmul(M_corrected, B) + if machine_config.camera == "FALCON": + vector_pixel = np.matmul(np.array([[-1, 0], [0, -1]]), vector_pixel) + + search_map_params.height_on_atlas = int( + search_map.height * search_map.pixel_size / dcg.atlas_pixel_size + ) + search_map_params.width_on_atlas = int( + search_map.width * search_map.pixel_size / dcg.atlas_pixel_size + ) + search_map_params.x_location = ( + vector_pixel[0] + / dcg.atlas_binning + * search_map.pixel_size + / dcg.atlas_pixel_size + + 2003 + ) + search_map_params.y_location = ( + vector_pixel[1] + / dcg.atlas_binning + * search_map.pixel_size + / dcg.atlas_pixel_size + + 2003 + ) + search_map.x_location = search_map_params.x_location + search_map.y_location = search_map_params.y_location + if _transport_object: + _transport_object.do_update_search_map(search_map.id, search_map_params) + murfey_db.add(search_map) + murfey_db.commit() + murfey_db.close() + + +def register_batch_position_in_database( + session_id: MurfeySessionID, + batch_name: str, + batch_parameters: BatchPositionParameters, + murfey_db: Session, +): + pass From 658f21d5236873ae0e9b50b413a4b9964c29d7df Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Jun 2025 12:13:29 +0100 Subject: [PATCH 03/32] Correct db propagation --- src/murfey/util/db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index d2161f168..d9c8a542c 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -96,6 +96,9 @@ class Session(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="session", sa_relationship_kwargs={"cascade": "delete"} ) + search_maps: List["SearchMap"] = Relationship( + back_populates="session", sa_relationship_kwargs={"cascade": "delete"} + ) rsync_instances: List[RsyncInstance] = Relationship( back_populates="session", sa_relationship_kwargs={"cascade": "delete"} ) @@ -617,7 +620,7 @@ class SearchMap(SQLModel, table=True): # type: ignore reference_matrix: Optional[dict[str, float]] = None stage_correction: Optional[dict[str, float]] = None image_shift_correction: Optional[dict[str, float]] = None - session: Optional[Session] = Relationship(back_populates="grid_squares") + session: Optional[Session] = Relationship(back_populates="search_maps") class Movie(SQLModel, table=True): # type: ignore From 20e74de171b2d07e758a318acb9b11afd5bbf1b2 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Jun 2025 12:13:55 +0100 Subject: [PATCH 04/32] Camera setup --- src/murfey/util/tomo_metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index 3ab8ca663..19cf61d1b 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -3,6 +3,7 @@ from murfey.server import _transport_object from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID +from murfey.server.gain import Camera from murfey.util.config import get_machine_config from murfey.util.db import DataCollectionGroup, SearchMap from murfey.util.db import Session as MurfeySession @@ -128,7 +129,9 @@ def register_search_map_in_database( ) M_corrected = np.matmul(np.linalg.inv(M), R) vector_pixel = np.matmul(M_corrected, B) - if machine_config.camera == "FALCON": + + camera = getattr(Camera, machine_config.camera) + if camera == Camera.FALCON: vector_pixel = np.matmul(np.array([[-1, 0], [0, -1]]), vector_pixel) search_map_params.height_on_atlas = int( From 405d259ec0a27fd6fe946a95653613c391607cd5 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Jun 2025 16:50:11 +0100 Subject: [PATCH 05/32] Register positions of tomograms on search maps --- src/murfey/client/contexts/tomo_metadata.py | 12 ++- src/murfey/server/api/workflow.py | 3 + src/murfey/server/feedback.py | 3 + src/murfey/util/db.py | 10 ++ src/murfey/util/models.py | 6 +- src/murfey/util/tomo_metadata.py | 114 ++++++++++++++++---- 6 files changed, 123 insertions(+), 25 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index e087d6ab8..b17938914 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -277,7 +277,9 @@ def post_transfer( "tag": visitless_source, "x_stage_position": batch_stage_location_x, "y_stage_position": batch_stage_location_y, - "search_map": search_map_name, + "x_beamshift": 0, + "y_beamshift": 0, + "search_map_name": search_map_name, }, ) @@ -298,8 +300,10 @@ def post_transfer( bp_url, json={ "tag": visitless_source, - "x_stage_position": beamshift_position_x, - "y_stage_position": beamshift_position_y, - "search_map": search_map_name, + "x_stage_position": batch_stage_location_x, + "y_stage_position": batch_stage_location_y, + "x_beamshift": beamshift_position_x, + "y_beamshift": beamshift_position_y, + "search_map_name": search_map_name, }, ) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index d40b2ac5a..618233866 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -819,6 +819,9 @@ def register_completed_tilt_series( "pixel_size": preproc_params.pixel_size, "manual_tilt_offset": -tilt_offset, "node_creator_queue": machine_config.node_creator_queue, + "search_map_id": ts.search_map_id, + "x_location": ts.x_location, + "y_location": ts.y_location, }, } if _transport_object: diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 80108bd87..24b67a2e2 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -1937,6 +1937,9 @@ def feedback_callback(header: dict, message: dict) -> None: "pixel_size": preproc_params.pixel_size, "manual_tilt_offset": -tilt_offset, "node_creator_queue": machine_config.node_creator_queue, + "search_map_id": relevant_tilt_series.search_map_id, + "x_location": relevant_tilt_series.x_location, + "y_location": relevant_tilt_series.y_location, }, } if murfey.server._transport_object: diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index d9c8a542c..0308bd5ae 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -349,15 +349,20 @@ class SessionProcessingParameters(SQLModel, table=True): # type: ignore class TiltSeries(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True) + ispyb_id: Optional[int] = None tag: str rsync_source: str session_id: int = Field(foreign_key="session.id") + search_map_id: int = Field(foreign_key="searchmap.id") tilt_series_length: int = -1 processing_requested: bool = False + x_location: Optional[float] = None + y_location: Optional[float] = None session: Optional[Session] = Relationship(back_populates="tilt_series") tilts: List["Tilt"] = Relationship( back_populates="tilt_series", sa_relationship_kwargs={"cascade": "delete"} ) + search_map: Optional["GridSquare"] = Relationship(back_populates="batch_positions") class Tilt(SQLModel, table=True): # type: ignore @@ -620,7 +625,12 @@ class SearchMap(SQLModel, table=True): # type: ignore reference_matrix: Optional[dict[str, float]] = None stage_correction: Optional[dict[str, float]] = None image_shift_correction: Optional[dict[str, float]] = None + width: Optional[int] = None + height: Optional[int] = None session: Optional[Session] = Relationship(back_populates="search_maps") + batch_positions: List["TiltSeries"] = Relationship( + back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} + ) class Movie(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index c299f1c75..00c917576 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -157,8 +157,10 @@ class SearchMapParameters: class BatchPositionParameters: tag: str x_stage_position: float - Y_stage_position: float - search_map: str + y_stage_position: float + x_beamshift: float + y_beamshift: float + search_map_name: str class MultigridWatcherSetup(BaseModel): diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index 19cf61d1b..bf60915a5 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -1,3 +1,5 @@ +import logging + import numpy as np from sqlmodel import Session, select @@ -7,8 +9,11 @@ from murfey.util.config import get_machine_config from murfey.util.db import DataCollectionGroup, SearchMap from murfey.util.db import Session as MurfeySession +from murfey.util.db import TiltSeries from murfey.util.models import BatchPositionParameters, SearchMapParameters +logger = logging.getLogger("murfey.client.util.tomo_metadata") + def register_search_map_in_database( session_id: MurfeySessionID, @@ -127,12 +132,13 @@ def register_search_map_in_database( ], ] ) - M_corrected = np.matmul(np.linalg.inv(M), R) - vector_pixel = np.matmul(M_corrected, B) + vector_pixel = np.matmul(np.linalg.inv(M), np.matmul(R, np.matmul(M, B))) camera = getattr(Camera, machine_config.camera) - if camera == Camera.FALCON: - vector_pixel = np.matmul(np.array([[-1, 0], [0, -1]]), vector_pixel) + if camera == Camera.FALCON or Camera.K3_FLIPY: + vector_pixel = np.matmul(np.array([[1, 0], [0, -1]]), vector_pixel) + elif camera == Camera.K3_FLIPX: + vector_pixel = np.matmul(np.array([[-1, 0], [0, 1]]), vector_pixel) search_map_params.height_on_atlas = int( search_map.height * search_map.pixel_size / dcg.atlas_pixel_size @@ -140,20 +146,8 @@ def register_search_map_in_database( search_map_params.width_on_atlas = int( search_map.width * search_map.pixel_size / dcg.atlas_pixel_size ) - search_map_params.x_location = ( - vector_pixel[0] - / dcg.atlas_binning - * search_map.pixel_size - / dcg.atlas_pixel_size - + 2003 - ) - search_map_params.y_location = ( - vector_pixel[1] - / dcg.atlas_binning - * search_map.pixel_size - / dcg.atlas_pixel_size - + 2003 - ) + search_map_params.x_location = vector_pixel[0] / dcg.atlas_pixel_size + 2003 + search_map_params.y_location = vector_pixel[1] / dcg.atlas_pixel_size + 2003 search_map.x_location = search_map_params.x_location search_map.y_location = search_map_params.y_location if _transport_object: @@ -169,4 +163,86 @@ def register_batch_position_in_database( batch_parameters: BatchPositionParameters, murfey_db: Session, ): - pass + search_map = murfey_db.exec( + select(SearchMap) + .where(SearchMap.name == batch_parameters.search_map_name) + .where(SearchMap.tag == batch_parameters.tag) + .where(SearchMap.session_id == session_id) + ).one() + + try: + tilt_series = murfey_db.exec( + select(TiltSeries) + .where(TiltSeries.tag == batch_name) + .where(TiltSeries.session_id == session_id) + ).one() + if tilt_series.x_location: + logger.info(f"Already did position analysis for tomogram {batch_name}") + return + except Exception: + tilt_series = TiltSeries( + tag=batch_name, + rsync_source=batch_parameters.tag, + session_id=session_id, + search_map_id=search_map.id, + ) + + # Get the pixel location on the searchmap + M = np.array( + [ + [ + search_map.reference_matrix["m11"], + search_map.reference_matrix["m12"], + ], + [ + search_map.reference_matrix["m21"], + search_map.reference_matrix["m22"], + ], + ] + ) + R1 = np.array( + [ + [ + search_map.stage_correction["m11"], + search_map.stage_correction["m12"], + ], + [ + search_map.stage_correction["m21"], + search_map.stage_correction["m22"], + ], + ] + ) + R2 = np.array( + [ + [ + search_map.image_shift_correction["m11"], + search_map.image_shift_correction["m12"], + ], + [ + search_map.image_shift_correction["m21"], + search_map.image_shift_correction["m22"], + ], + ] + ) + + A = np.array([search_map.x_stage_position, search_map.y_stage_position]) + B = np.array([batch_parameters.x_stage_position, batch_parameters.y_stage_position]) + + vector_pixel = np.matmul( + np.linalg.inv(M), + np.matmul(np.linalg.inv(R1), np.matmul(np.linalg.inv(R2), np.matmul(M, B - A))), + ) + centre_batch_pixel = vector_pixel / search_map.pixel_size + [ + search_map.width / 2, + search_map.height / 2, + ] + tilt_series.x_location = ( + centre_batch_pixel[0] + - BatchPositionParameters.x_beamshift / search_map.pixel_size + ) + tilt_series.y_location = ( + centre_batch_pixel[1] + - BatchPositionParameters.y_beamshift / search_map.pixel_size + ) + murfey_db.add(tilt_series) + murfey_db.commit() From 1167df7978f0b0abcd639afe6de06d3da2019e58 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 09:30:56 +0100 Subject: [PATCH 06/32] Method to enter tomo metadata context --- src/murfey/client/analyser.py | 11 ++++++++++- src/murfey/client/contexts/tomo_metadata.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index dcebfa607..fde420908 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -19,6 +19,7 @@ from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext from murfey.client.contexts.tomo import TomographyContext +from murfey.client.contexts.tomo_metadata import TomographyMetadataContext from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.client.rsync import RSyncerUpdate, TransferResult from murfey.util.client import Observer, get_machine_config_client @@ -226,6 +227,13 @@ def _analyse(self): and not self._context ): self._context = SPAMetadataContext("epu", self._basepath) + elif ( + "Batch" in transferred_file.parts + or "SearchMap" in transferred_file.parts + or transferred_file.name == "Session.dm" + and not self._context + ): + self._context = TomographyMetadataContext("tomo", self._basepath) self.post_transfer(transferred_file) else: dc_metadata = {} @@ -369,9 +377,10 @@ def _analyse(self): elif isinstance( self._context, ( - TomographyContext, SPAModularContext, SPAMetadataContext, + TomographyContext, + TomographyMetadataContext, ), ): context = str(self._context).split(" ")[0].split(".")[-1] diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index b17938914..82888499b 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -50,7 +50,7 @@ def post_transfer( **kwargs, ) - if transferred_file.name == "Session.xml" and environment: + if transferred_file.name == "Session.dm" and environment: logger.info("Tomography session metadata found") with open(transferred_file, "r") as session_xml: session_data = xmltodict.parse(session_xml.read()) From 277eb8c9b7c62417634b481fc4e8a88c52275010 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 09:55:44 +0100 Subject: [PATCH 07/32] Database will only allow simple types --- src/murfey/client/contexts/tomo_metadata.py | 4 +- src/murfey/util/db.py | 15 +- src/murfey/util/models.py | 6 +- src/murfey/util/tomo_metadata.py | 214 +++++++++++++------- 4 files changed, 163 insertions(+), 76 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 82888499b..73628f7ae 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -175,7 +175,9 @@ def post_transfer( "m22": float(key_val["a:Value"]["b:_m22"]), } if not stage_matrix or not image_matrix: - print("No matrix found") + logger.error( + f"No stage or image shift matrix found for {transferred_file}" + ) ref_matrix = { "m11": float( diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 0308bd5ae..a32a7dec6 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -622,9 +622,18 @@ class SearchMap(SQLModel, table=True): # type: ignore pixel_size: Optional[float] = None image: str = "" binning: Optional[float] = None - reference_matrix: Optional[dict[str, float]] = None - stage_correction: Optional[dict[str, float]] = None - image_shift_correction: Optional[dict[str, float]] = None + reference_matrix_m11: Optional[float] = None + reference_matrix_m12: Optional[float] = None + reference_matrix_m21: Optional[float] = None + reference_matrix_m22: Optional[float] = None + stage_correction_m11: Optional[float] = None + stage_correction_m12: Optional[float] = None + stage_correction_m21: Optional[float] = None + stage_correction_m22: Optional[float] = None + image_shift_correction_m11: Optional[float] = None + image_shift_correction_m12: Optional[float] = None + image_shift_correction_m21: Optional[float] = None + image_shift_correction_m22: Optional[float] = None width: Optional[int] = None height: Optional[int] = None session: Optional[Session] = Relationship(back_populates="search_maps") diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 00c917576..12901d5f3 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -145,9 +145,9 @@ class SearchMapParameters: pixel_size: Optional[float] = None image: Optional[str] = None binning: Optional[float] = None - reference_matrix: Optional[dict[str, float]] = None - stage_correction: Optional[dict[str, float]] = None - image_shift_correction: Optional[dict[str, float]] = None + reference_matrix: dict[str, float] = {} + stage_correction: dict[str, float] = {} + image_shift_correction: dict[str, float] = {} height: Optional[int] = None width: Optional[int] = None height_on_atlas: Optional[int] = None diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index bf60915a5..8742c8276 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -42,15 +42,53 @@ def register_search_map_in_database( search_map.pixel_size = search_map_params.pixel_size or search_map.pixel_size search_map.image = search_map_params.image or search_map.image search_map.binning = search_map_params.binning or search_map.binning - search_map.reference_matrix = ( - search_map_params.reference_matrix or search_map.reference_matrix + search_map.reference_matrix_m11 = ( + search_map_params.reference_matrix.get("m11") + or search_map.reference_matrix_m11 ) - search_map.stage_correction = ( - search_map_params.stage_correction or search_map.stage_correction + search_map.reference_matrix_m12 = ( + search_map_params.reference_matrix.get("m12") + or search_map.reference_matrix_m12 ) - search_map.image_shift_correction = ( - search_map_params.image_shift_correction - or search_map.image_shift_correction + search_map.reference_matrix_m21 = ( + search_map_params.reference_matrix.get("m21") + or search_map.reference_matrix_m21 + ) + search_map.reference_matrix_m22 = ( + search_map_params.reference_matrix.get("m22") + or search_map.reference_matrix_m22 + ) + search_map.stage_correction_m11 = ( + search_map_params.stage_correction.get("m11") + or search_map.stage_correction_m11 + ) + search_map.stage_correction_m12 = ( + search_map_params.stage_correction.get("m12") + or search_map.stage_correction_m12 + ) + search_map.stage_correction_m21 = ( + search_map_params.stage_correction.get("m21") + or search_map.stage_correction_m21 + ) + search_map.stage_correction_m22 = ( + search_map_params.stage_correction.get("m22") + or search_map.stage_correction_m22 + ) + search_map.image_shift_correction_m11 = ( + search_map_params.image_shift_correction.get("m11") + or search_map.image_shift_correction_m11 + ) + search_map.image_shift_correction_m12 = ( + search_map_params.image_shift_correction.get("m12") + or search_map.image_shift_correction_m12 + ) + search_map.image_shift_correction_m21 = ( + search_map_params.image_shift_correction.get("m21") + or search_map.image_shift_correction_m21 + ) + search_map.image_shift_correction_m22 = ( + search_map_params.image_shift_correction.get("m22") + or search_map.image_shift_correction_m22 ) search_map.height = search_map_params.height or search_map.height search_map.width = search_map_params.width or search_map.width @@ -78,9 +116,26 @@ def register_search_map_in_database( pixel_size=search_map_params.pixel_size, image=search_map_params.image, binning=search_map_params.binning, - reference_matrix=search_map_params.reference_matrix, - stage_correction=search_map_params.stage_correction, - image_shift_correction=search_map_params.image_shift_correction, + reference_matrix_m11=search_map_params.reference_matrix.get("m11"), + reference_matrix_m12=search_map_params.reference_matrix.get("m12"), + reference_matrix_m21=search_map_params.reference_matrix.get("m21"), + reference_matrix_m22=search_map_params.reference_matrix.get("m22"), + stage_correction_m11=search_map_params.stage_correction.get("m11"), + stage_correction_m12=search_map_params.stage_correction.get("m12"), + stage_correction_m21=search_map_params.stage_correction.get("m21"), + stage_correction_m22=search_map_params.stage_correction.get("m22"), + image_shift_correction_m11=search_map_params.image_shift_correction.get( + "m11" + ), + image_shift_correction_m12=search_map_params.image_shift_correction.get( + "m12" + ), + image_shift_correction_m21=search_map_params.image_shift_correction.get( + "m21" + ), + image_shift_correction_m22=search_map_params.image_shift_correction.get( + "m22" + ), height=search_map_params.height, width=search_map_params.width, ) @@ -96,8 +151,8 @@ def register_search_map_in_database( if all( [ - search_map.reference_matrix, - search_map.stage_correction, + search_map.reference_matrix_m11, + search_map.stage_correction_m11, search_map.x_stage_position, search_map.y_stage_position, search_map.pixel_size, @@ -110,12 +165,12 @@ def register_search_map_in_database( M = np.array( [ [ - search_map.reference_matrix["m11"], - search_map.reference_matrix["m12"], + search_map.reference_matrix_m11, + search_map.reference_matrix_m12, ], [ - search_map.reference_matrix["m21"], - search_map.reference_matrix["m22"], + search_map.reference_matrix_m21, + search_map.reference_matrix_m22, ], ] ) @@ -123,12 +178,12 @@ def register_search_map_in_database( R = np.array( [ [ - search_map.stage_correction["m11"], - search_map.stage_correction["m12"], + search_map.stage_correction_m11, + search_map.stage_correction_m12, ], [ - search_map.stage_correction["m21"], - search_map.stage_correction["m22"], + search_map.stage_correction_m21, + search_map.stage_correction_m22, ], ] ) @@ -152,6 +207,8 @@ def register_search_map_in_database( search_map.y_location = search_map_params.y_location if _transport_object: _transport_object.do_update_search_map(search_map.id, search_map_params) + else: + logger.info(f"Unable to register search map {search_map_name} position yet") murfey_db.add(search_map) murfey_db.commit() murfey_db.close() @@ -188,61 +245,80 @@ def register_batch_position_in_database( ) # Get the pixel location on the searchmap - M = np.array( + if all( [ - [ - search_map.reference_matrix["m11"], - search_map.reference_matrix["m12"], - ], - [ - search_map.reference_matrix["m21"], - search_map.reference_matrix["m22"], - ], + search_map.reference_matrix_m11, + search_map.stage_correction_m11, + search_map.x_stage_position, + search_map.y_stage_position, + search_map.pixel_size, + search_map.height, + search_map.width, ] - ) - R1 = np.array( - [ - [ - search_map.stage_correction["m11"], - search_map.stage_correction["m12"], - ], + ): + M = np.array( [ - search_map.stage_correction["m21"], - search_map.stage_correction["m22"], - ], - ] - ) - R2 = np.array( - [ + [ + search_map.reference_matrix_m11, + search_map.reference_matrix_m12, + ], + [ + search_map.reference_matrix_m21, + search_map.reference_matrix_m22, + ], + ] + ) + R1 = np.array( [ - search_map.image_shift_correction["m11"], - search_map.image_shift_correction["m12"], - ], + [ + search_map.stage_correction_m11, + search_map.stage_correction_m12, + ], + [ + search_map.stage_correction_m21, + search_map.stage_correction_m22, + ], + ] + ) + R2 = np.array( [ - search_map.image_shift_correction["m21"], - search_map.image_shift_correction["m22"], - ], - ] - ) + [ + search_map.image_shift_correction_m11, + search_map.image_shift_correction_m12, + ], + [ + search_map.image_shift_correction_m21, + search_map.image_shift_correction_m22, + ], + ] + ) - A = np.array([search_map.x_stage_position, search_map.y_stage_position]) - B = np.array([batch_parameters.x_stage_position, batch_parameters.y_stage_position]) + A = np.array([search_map.x_stage_position, search_map.y_stage_position]) + B = np.array( + [batch_parameters.x_stage_position, batch_parameters.y_stage_position] + ) - vector_pixel = np.matmul( - np.linalg.inv(M), - np.matmul(np.linalg.inv(R1), np.matmul(np.linalg.inv(R2), np.matmul(M, B - A))), - ) - centre_batch_pixel = vector_pixel / search_map.pixel_size + [ - search_map.width / 2, - search_map.height / 2, - ] - tilt_series.x_location = ( - centre_batch_pixel[0] - - BatchPositionParameters.x_beamshift / search_map.pixel_size - ) - tilt_series.y_location = ( - centre_batch_pixel[1] - - BatchPositionParameters.y_beamshift / search_map.pixel_size - ) + vector_pixel = np.matmul( + np.linalg.inv(M), + np.matmul( + np.linalg.inv(R1), np.matmul(np.linalg.inv(R2), np.matmul(M, B - A)) + ), + ) + centre_batch_pixel = vector_pixel / search_map.pixel_size + [ + search_map.width / 2, + search_map.height / 2, + ] + tilt_series.x_location = ( + centre_batch_pixel[0] + - BatchPositionParameters.x_beamshift / search_map.pixel_size + ) + tilt_series.y_location = ( + centre_batch_pixel[1] + - BatchPositionParameters.y_beamshift / search_map.pixel_size + ) + else: + logger.warning( + f"No search map information available to register position of {batch_name}" + ) murfey_db.add(tilt_series) murfey_db.commit() From fe44f9a06e2de484315b6bf218d766c16b73518f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 09:59:34 +0100 Subject: [PATCH 08/32] Clearer naming --- src/murfey/util/tomo_metadata.py | 49 ++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index 8742c8276..d6d1a7cca 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -162,7 +162,7 @@ def register_search_map_in_database( dcg.atlas_binning, ] ): - M = np.array( + reference_shift_matrix = np.array( [ [ search_map.reference_matrix_m11, @@ -174,8 +174,10 @@ def register_search_map_in_database( ], ] ) - B = np.array([search_map.x_stage_position, search_map.y_stage_position]) - R = np.array( + stage_vector = np.array( + [search_map.x_stage_position, search_map.y_stage_position] + ) + stage_correction_matrix = np.array( [ [ search_map.stage_correction_m11, @@ -187,13 +189,18 @@ def register_search_map_in_database( ], ] ) - vector_pixel = np.matmul(np.linalg.inv(M), np.matmul(R, np.matmul(M, B))) + corrected_vector = np.matmul( + np.linalg.inv(reference_shift_matrix), + np.matmul( + stage_correction_matrix, np.matmul(reference_shift_matrix, stage_vector) + ), + ) camera = getattr(Camera, machine_config.camera) if camera == Camera.FALCON or Camera.K3_FLIPY: - vector_pixel = np.matmul(np.array([[1, 0], [0, -1]]), vector_pixel) + corrected_vector = np.matmul(np.array([[1, 0], [0, -1]]), corrected_vector) elif camera == Camera.K3_FLIPX: - vector_pixel = np.matmul(np.array([[-1, 0], [0, 1]]), vector_pixel) + corrected_vector = np.matmul(np.array([[-1, 0], [0, 1]]), corrected_vector) search_map_params.height_on_atlas = int( search_map.height * search_map.pixel_size / dcg.atlas_pixel_size @@ -201,8 +208,8 @@ def register_search_map_in_database( search_map_params.width_on_atlas = int( search_map.width * search_map.pixel_size / dcg.atlas_pixel_size ) - search_map_params.x_location = vector_pixel[0] / dcg.atlas_pixel_size + 2003 - search_map_params.y_location = vector_pixel[1] / dcg.atlas_pixel_size + 2003 + search_map_params.x_location = corrected_vector[0] / dcg.atlas_pixel_size + 2003 + search_map_params.y_location = corrected_vector[1] / dcg.atlas_pixel_size + 2003 search_map.x_location = search_map_params.x_location search_map.y_location = search_map_params.y_location if _transport_object: @@ -256,7 +263,7 @@ def register_batch_position_in_database( search_map.width, ] ): - M = np.array( + reference_shift_matrix = np.array( [ [ search_map.reference_matrix_m11, @@ -268,7 +275,7 @@ def register_batch_position_in_database( ], ] ) - R1 = np.array( + stage_correction_matrix = np.array( [ [ search_map.stage_correction_m11, @@ -280,7 +287,7 @@ def register_batch_position_in_database( ], ] ) - R2 = np.array( + image_shift_matrix = np.array( [ [ search_map.image_shift_correction_m11, @@ -293,18 +300,24 @@ def register_batch_position_in_database( ] ) - A = np.array([search_map.x_stage_position, search_map.y_stage_position]) - B = np.array( - [batch_parameters.x_stage_position, batch_parameters.y_stage_position] + stage_vector = np.array( + [ + batch_parameters.x_stage_position - search_map.x_stage_position, + batch_parameters.y_stage_position - search_map.y_stage_position, + ] ) - vector_pixel = np.matmul( - np.linalg.inv(M), + corrected_vector = np.matmul( + np.linalg.inv(reference_shift_matrix), np.matmul( - np.linalg.inv(R1), np.matmul(np.linalg.inv(R2), np.matmul(M, B - A)) + np.linalg.inv(stage_correction_matrix), + np.matmul( + np.linalg.inv(image_shift_matrix), + np.matmul(reference_shift_matrix, stage_vector), + ), ), ) - centre_batch_pixel = vector_pixel / search_map.pixel_size + [ + centre_batch_pixel = corrected_vector / search_map.pixel_size + [ search_map.width / 2, search_map.height / 2, ] From 2e3d2fd11d36c5f08ca99df50b0a040ae15b85a8 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 10:07:15 +0100 Subject: [PATCH 09/32] Revert to Dict --- src/murfey/util/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 12901d5f3..154999248 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -145,9 +145,9 @@ class SearchMapParameters: pixel_size: Optional[float] = None image: Optional[str] = None binning: Optional[float] = None - reference_matrix: dict[str, float] = {} - stage_correction: dict[str, float] = {} - image_shift_correction: dict[str, float] = {} + reference_matrix: Dict[str, float] = {} + stage_correction: Dict[str, float] = {} + image_shift_correction: Dict[str, float] = {} height: Optional[int] = None width: Optional[int] = None height_on_atlas: Optional[int] = None From 97bf289ec8f83e2a37027d25dada2534bf0c4d84 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 10:10:42 +0100 Subject: [PATCH 10/32] Forgot the BaseModel --- src/murfey/util/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 154999248..d82551d1f 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -136,7 +136,7 @@ class FoilHoleParameters(BaseModel): diameter: Optional[float] = None -class SearchMapParameters: +class SearchMapParameters(BaseModel): tag: str x_location: Optional[float] = None y_location: Optional[float] = None @@ -154,7 +154,7 @@ class SearchMapParameters: width_on_atlas: Optional[int] = None -class BatchPositionParameters: +class BatchPositionParameters(BaseModel): tag: str x_stage_position: float y_stage_position: float From 72201ece806b5fbeb32ca5d8e54fb4952337a5da Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 10:19:01 +0100 Subject: [PATCH 11/32] DB relationships were set up wrong --- src/murfey/util/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index a32a7dec6..8cf39e6ab 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -362,7 +362,7 @@ class TiltSeries(SQLModel, table=True): # type: ignore tilts: List["Tilt"] = Relationship( back_populates="tilt_series", sa_relationship_kwargs={"cascade": "delete"} ) - search_map: Optional["GridSquare"] = Relationship(back_populates="batch_positions") + search_map: Optional["SearchMap"] = Relationship(back_populates="tilt_series") class Tilt(SQLModel, table=True): # type: ignore @@ -637,7 +637,7 @@ class SearchMap(SQLModel, table=True): # type: ignore width: Optional[int] = None height: Optional[int] = None session: Optional[Session] = Relationship(back_populates="search_maps") - batch_positions: List["TiltSeries"] = Relationship( + tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) From f20e3fd6eb66b6a554c06a46d3d8f1e3d88d7b2c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 13:37:12 +0100 Subject: [PATCH 12/32] Sanitise inputs --- src/murfey/util/tomo_metadata.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index d6d1a7cca..802802aa3 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -6,6 +6,7 @@ from murfey.server import _transport_object from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID from murfey.server.gain import Camera +from murfey.util import sanitise from murfey.util.config import get_machine_config from murfey.util.db import DataCollectionGroup, SearchMap from murfey.util.db import Session as MurfeySession @@ -215,7 +216,9 @@ def register_search_map_in_database( if _transport_object: _transport_object.do_update_search_map(search_map.id, search_map_params) else: - logger.info(f"Unable to register search map {search_map_name} position yet") + logger.info( + f"Unable to register search map {sanitise(search_map_name)} position yet" + ) murfey_db.add(search_map) murfey_db.commit() murfey_db.close() @@ -241,7 +244,9 @@ def register_batch_position_in_database( .where(TiltSeries.session_id == session_id) ).one() if tilt_series.x_location: - logger.info(f"Already did position analysis for tomogram {batch_name}") + logger.info( + f"Already did position analysis for tomogram {sanitise(batch_name)}" + ) return except Exception: tilt_series = TiltSeries( @@ -331,7 +336,7 @@ def register_batch_position_in_database( ) else: logger.warning( - f"No search map information available to register position of {batch_name}" + f"No search map information available to register position of {sanitise(batch_name)}" ) murfey_db.add(tilt_series) murfey_db.commit() From 58dd306a45d04461790188c12acc74807c99dfac Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 13:37:26 +0100 Subject: [PATCH 13/32] No Images-Disc for tomo --- src/murfey/client/contexts/tomo_metadata.py | 45 ++++----------------- 1 file changed, 8 insertions(+), 37 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 73628f7ae..16f803992 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -17,22 +17,6 @@ requests.get, requests.post, requests.put, requests.delete = authorised_requests() -def get_visitless_source( - transferred_file: Path, environment: MurfeyInstanceEnvironment -) -> Optional[str]: - source = _get_source(transferred_file, environment=environment) - visitless_source_search_dir = str(source).replace(f"/{environment.visit}", "") - visitless_source_images_dirs = sorted( - Path(visitless_source_search_dir).glob("Images-Disc*"), - key=lambda x: x.stat().st_ctime, - ) - if not visitless_source_images_dirs: - logger.warning(f"Cannot find Images-Disc* in {visitless_source_search_dir}") - return None - visitless_source = str(visitless_source_images_dirs[-1]) - return visitless_source - - class TomographyMetadataContext(Context): def __init__(self, acquisition_software: str, basepath: Path): super().__init__("Tomography_metadata", acquisition_software) @@ -102,22 +86,9 @@ def post_transfer( atlas=Path(partial_path), sample=sample ) url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}" - dcg_search_dir = "/".join( + dcg_tag = "/".join( p for p in transferred_file.parent.parts if p != environment.visit ) - dcg_search_dir = ( - dcg_search_dir[1:] - if dcg_search_dir.startswith("//") - else dcg_search_dir - ) - dcg_images_dirs = sorted( - Path(dcg_search_dir).glob("Images-Disc*"), - key=lambda x: x.stat().st_ctime, - ) - if not dcg_images_dirs: - logger.warning(f"Cannot find Images-Disc* in {dcg_search_dir}") - return - dcg_tag = str(dcg_images_dirs[-1]) dcg_data = { "experiment_type": "tomo", "experiment_type_id": 36, @@ -202,12 +173,8 @@ def post_transfer( ), } - visitless_source = get_visitless_source(transferred_file, environment) - if not visitless_source: - return - - sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" source = _get_source(transferred_file, environment=environment) + visitless_source = str(source).replace(f"/{environment.visit}", "") image_path = ( _file_transferred_to( environment, source, transferred_file.parent / "SearchMap.jpg" @@ -215,6 +182,8 @@ def post_transfer( if source else "" ) + + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" capture_post( sm_url, json={ @@ -235,7 +204,8 @@ def post_transfer( with open(transferred_file, "r") as sm_xml: sm_data = xmltodict.parse(sm_xml.read()) - visitless_source = get_visitless_source(transferred_file, environment) + source = _get_source(transferred_file, environment=environment) + visitless_source = str(source).replace(f"/{environment.visit}", "") if not visitless_source: return @@ -257,7 +227,8 @@ def post_transfer( with open(transferred_file) as xml: for_parsing = xml.read() batch_xml = xmltodict.parse(for_parsing) - visitless_source = get_visitless_source(transferred_file, environment) + source = _get_source(transferred_file, environment=environment) + visitless_source = str(source).replace(f"/{environment.visit}", "") if not visitless_source: return From 9824baf2b56d39a2e0a1ee14e31d1c11dc996538 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 14:07:15 +0100 Subject: [PATCH 14/32] Add new routers to manifest --- src/murfey/util/route_manifest.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index e8ca73ef7..59ae64447 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -818,6 +818,21 @@ murfey.server.api.session_control.spa_router: type: int methods: - POST +murfey.server.api.session_control.tomography_router: + - path: /session_control/tomography/sessions/{session_id}/search_map/{sm_name} + function: register_search_map + path_params: + - name: sm_name + type: str + methods: + - POST + - path: /session_control/tomography/sessions/{session_id}/batch_position/{batch_name} + function: register_batch_position + path_params: + - name: batch_name + type: str + methods: + - POST murfey.server.api.session_info.correlative_router: - path: /session_info/correlative/sessions/{session_id}/upstream_visits function: find_upstream_visits From c436d342c9f8820a30abdc139c45b9ebcfcb63ea Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Jun 2025 15:41:07 +0100 Subject: [PATCH 15/32] Routing fixes --- src/murfey/client/contexts/tomo_metadata.py | 4 ++-- src/murfey/server/main.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 16f803992..c1f09c806 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -183,7 +183,7 @@ def post_transfer( else "" ) - sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( sm_url, json={ @@ -213,7 +213,7 @@ def post_transfer( sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) - sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.stem)}" + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( sm_url, json={ diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index d588476af..a0dba1e1a 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -81,6 +81,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.session_control.router) app.include_router(murfey.server.api.session_control.spa_router) +app.include_router(murfey.server.api.session_control.tomography_router) app.include_router(murfey.server.api.session_info.router) app.include_router(murfey.server.api.session_info.correlative_router) From 6ef2ef661a8c0493bde187c4b7216df384272777 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 30 Jun 2025 11:31:02 +0100 Subject: [PATCH 16/32] Ensure the existance of all higher levels before inserting tomo metadata --- src/murfey/client/contexts/tomo_metadata.py | 58 ++++++++++++++------- src/murfey/server/api/session_control.py | 10 ++-- src/murfey/server/main.py | 2 +- src/murfey/util/route_manifest.yaml | 6 +-- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index c1f09c806..3de5bcdb7 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -17,6 +17,22 @@ requests.get, requests.post, requests.put, requests.delete = authorised_requests() +def ensure_dcg_exists(transferred_file: Path, environment: MurfeyInstanceEnvironment): + # Make sure we have a data collection group + source = _get_source(transferred_file, environment=environment) + if not source: + return None + dcg_tag = str(source).replace(f"/{environment.visit}", "") + url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}" + dcg_data = { + "experiment_type": "single particle", + "experiment_type_id": 37, + "tag": dcg_tag, + } + capture_post(url, json=dcg_data) + return dcg_tag + + class TomographyMetadataContext(Context): def __init__(self, acquisition_software: str, basepath: Path): super().__init__("Tomography_metadata", acquisition_software) @@ -106,6 +122,7 @@ def post_transfer( elif transferred_file.name == "SearchMap.xml" and environment: logger.info("Tomography session search map xml found") + dcg_tag = ensure_dcg_exists(transferred_file, environment) with open(transferred_file, "r") as sm_xml: sm_data = xmltodict.parse(sm_xml.read()) @@ -174,7 +191,6 @@ def post_transfer( } source = _get_source(transferred_file, environment=environment) - visitless_source = str(source).replace(f"/{environment.visit}", "") image_path = ( _file_transferred_to( environment, source, transferred_file.parent / "SearchMap.jpg" @@ -183,11 +199,11 @@ def post_transfer( else "" ) - sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( sm_url, json={ - "tag": visitless_source, + "tag": dcg_tag, "x_stage_position": float(stage_position["X"]), "y_stage_position": float(stage_position["Y"]), "pixel_size": sm_pixel_size, @@ -201,36 +217,30 @@ def post_transfer( elif transferred_file.name == "SearchMap.dm" and environment: logger.info("Tomography session search map dm found") + dcg_tag = ensure_dcg_exists(transferred_file, environment) with open(transferred_file, "r") as sm_xml: sm_data = xmltodict.parse(sm_xml.read()) - source = _get_source(transferred_file, environment=environment) - visitless_source = str(source).replace(f"/{environment.visit}", "") - if not visitless_source: - return - # This bit gets SearchMap size sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) - sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( sm_url, json={ - "tag": visitless_source, + "tag": dcg_tag, "height": sm_height, "width": sm_width, }, ) elif transferred_file.name == "BatchPositionsList.xml" and environment: + logger.info("Tomography session batch positions list found") + dcg_tag = ensure_dcg_exists(transferred_file, environment) with open(transferred_file) as xml: for_parsing = xml.read() batch_xml = xmltodict.parse(for_parsing) - source = _get_source(transferred_file, environment=environment) - visitless_source = str(source).replace(f"/{environment.visit}", "") - if not visitless_source: - return for batch_position in batch_xml["BatchPositionsList"]["BatchPositions"][ "BatchPositionParameters" @@ -243,11 +253,22 @@ def post_transfer( batch_stage_location_y = float( batch_position["PositionOnTileSet"]["StagePositionY"] ) - bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=batch_name)}" + + # Always need search map before batch position + sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=search_map_name)}" + capture_post( + sm_url, + json={ + "tag": dcg_tag, + }, + ) + + # Then register batch position + bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=batch_name)}" capture_post( bp_url, json={ - "tag": visitless_source, + "tag": dcg_tag, "x_stage_position": batch_stage_location_x, "y_stage_position": batch_stage_location_y, "x_beamshift": 0, @@ -268,11 +289,12 @@ def post_transfer( beamshift_position_x = float(beamshift["PositionX"]) beamshift_position_y = float(beamshift["PositionY"]) - bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomography_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=beamshift_name)}" + # Registration of beamshifted position + bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=beamshift_name)}" capture_post( bp_url, json={ - "tag": visitless_source, + "tag": dcg_tag, "x_stage_position": batch_stage_location_x, "y_stage_position": batch_stage_location_y, "x_beamshift": beamshift_position_x, diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 8ef0dc87e..5ef15c140 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -370,14 +370,14 @@ def register_foil_hole( return _register_foil_hole(session_id, gs_name, foil_hole_params, db) -tomography_router = APIRouter( - prefix="/session_control/tomography", +tomo_router = APIRouter( + prefix="/session_control/tomo", dependencies=[Depends(validate_instrument_token)], - tags=["Session Control: Tomography"], + tags=["Session Control: CryoET"], ) -@tomography_router.post("/sessions/{session_id}/search_map/{sm_name}") +@tomo_router.post("/sessions/{session_id}/search_map/{sm_name}") def register_search_map( session_id: MurfeySessionID, sm_name: str, @@ -387,7 +387,7 @@ def register_search_map( return register_search_map_in_database(session_id, sm_name, search_map_params, db) -@tomography_router.post("/sessions/{session_id}/batch_position/{batch_name}") +@tomo_router.post("/sessions/{session_id}/batch_position/{batch_name}") def register_batch_position( session_id: MurfeySessionID, batch_name: str, diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index a0dba1e1a..e36b7d142 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -81,7 +81,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.session_control.router) app.include_router(murfey.server.api.session_control.spa_router) -app.include_router(murfey.server.api.session_control.tomography_router) +app.include_router(murfey.server.api.session_control.tomo_router) app.include_router(murfey.server.api.session_info.router) app.include_router(murfey.server.api.session_info.correlative_router) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 59ae64447..03bfb2af7 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -818,15 +818,15 @@ murfey.server.api.session_control.spa_router: type: int methods: - POST -murfey.server.api.session_control.tomography_router: - - path: /session_control/tomography/sessions/{session_id}/search_map/{sm_name} +murfey.server.api.session_control.tomo_router: + - path: /session_control/tomo/sessions/{session_id}/search_map/{sm_name} function: register_search_map path_params: - name: sm_name type: str methods: - POST - - path: /session_control/tomography/sessions/{session_id}/batch_position/{batch_name} + - path: /session_control/tomo/sessions/{session_id}/batch_position/{batch_name} function: register_batch_position path_params: - name: batch_name From 8adb4da1dc78d538434ddbb0212b03b6f34a3c26 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 30 Jun 2025 15:29:25 +0100 Subject: [PATCH 17/32] Fixes for posting metadata in the right order --- src/murfey/client/contexts/tomo_metadata.py | 45 ++++++++++----------- src/murfey/server/api/workflow.py | 4 ++ src/murfey/util/tomo_metadata.py | 29 +++++++------ 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 3de5bcdb7..0318c76ac 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -55,7 +55,7 @@ def post_transfer( with open(transferred_file, "r") as session_xml: session_data = xmltodict.parse(session_xml.read()) - windows_path = session_data["TomographySession"]["AtlasId"]["#text"] + windows_path = session_data["TomographySession"]["AtlasId"] logger.info(f"Windows path to atlas metadata found: {windows_path}") visit_index = windows_path.split("\\").index(environment.visit) partial_path = "/".join(windows_path.split("\\")[visit_index + 1 :]) @@ -97,28 +97,27 @@ def post_transfer( else: logger.warning(f"Sample could not be identified for {transferred_file}") return - if source: - environment.samples[source] = SampleInfo( - atlas=Path(partial_path), sample=sample - ) - url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}" - dcg_tag = "/".join( - p for p in transferred_file.parent.parts if p != environment.visit - ) - dcg_data = { - "experiment_type": "tomo", - "experiment_type_id": 36, - "tag": dcg_tag, - "atlas": str( - _atlas_destination(environment, source, transferred_file) - / environment.samples[source].atlas.parent - / atlas_xml_path.with_suffix(".jpg").name - ), - "sample": environment.samples[source].sample, - "atlas_pixel_size": atlas_pixel_size, - "atlas_binning": atlas_binning, - } - capture_post(url, json=dcg_data) + environment.samples[source] = SampleInfo( + atlas=Path(partial_path), sample=sample + ) + url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}" + dcg_tag = "/".join( + p for p in transferred_file.parent.parts if p != environment.visit + ).replace("//", "/") + dcg_data = { + "experiment_type": "tomo", + "experiment_type_id": 36, + "tag": dcg_tag, + "atlas": str( + _atlas_destination(environment, source, transferred_file) + / environment.samples[source].atlas.parent + / atlas_xml_path.with_suffix(".jpg").name + ), + "sample": environment.samples[source].sample, + "atlas_pixel_size": atlas_pixel_size, + "atlas_binning": atlas_binning, + } + capture_post(url, json=dcg_data) elif transferred_file.name == "SearchMap.xml" and environment: logger.info("Tomography session search map xml found") diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 618233866..2976284fa 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -108,6 +108,9 @@ def register_dc_group( dcg_murfey[0].atlas_pixel_size = ( dcg_params.atlas_pixel_size or dcg_murfey[0].atlas_pixel_size ) + dcg_murfey[0].atlas_binning = ( + dcg_params.atlas_binning or dcg_murfey[0].atlas_binning + ) if _transport_object: if dcg_murfey[0].atlas_id is not None: @@ -148,6 +151,7 @@ def register_dc_group( "atlas": dcg_params.atlas, "sample": dcg_params.sample, "atlas_pixel_size": dcg_params.atlas_pixel_size, + "atlas_binning": dcg_params.atlas_binning, } if _transport_object: diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index 802802aa3..e07c15339 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -95,7 +95,8 @@ def register_search_map_in_database( search_map.width = search_map_params.width or search_map.width if _transport_object: _transport_object.do_update_search_map(search_map.id, search_map_params) - except Exception: + except Exception as e: + logger.info(f"Registering new search map due to {e}", exc_info=True) if _transport_object: sm_ispyb_response = _transport_object.do_insert_search_map( dcg.atlas_id, search_map_params @@ -140,8 +141,6 @@ def register_search_map_in_database( height=search_map_params.height, width=search_map_params.width, ) - murfey_db.add(search_map) - murfey_db.commit() murfey_session = murfey_db.exec( select(MurfeySession).where(MurfeySession.id == session_id) @@ -149,7 +148,6 @@ def register_search_map_in_database( machine_config = get_machine_config(instrument_name=murfey_session.instrument_name)[ murfey_session.instrument_name ] - if all( [ search_map.reference_matrix_m11, @@ -209,15 +207,22 @@ def register_search_map_in_database( search_map_params.width_on_atlas = int( search_map.width * search_map.pixel_size / dcg.atlas_pixel_size ) - search_map_params.x_location = corrected_vector[0] / dcg.atlas_pixel_size + 2003 - search_map_params.y_location = corrected_vector[1] / dcg.atlas_pixel_size + 2003 + search_map_params.x_location = float( + corrected_vector[0] / dcg.atlas_pixel_size + 2003 + ) + search_map_params.y_location = float( + corrected_vector[1] / dcg.atlas_pixel_size + 2003 + ) search_map.x_location = search_map_params.x_location search_map.y_location = search_map_params.y_location if _transport_object: _transport_object.do_update_search_map(search_map.id, search_map_params) else: logger.info( - f"Unable to register search map {sanitise(search_map_name)} position yet" + f"Unable to register search map {sanitise(search_map_name)} position yet: " + f"stage {search_map_params.x_stage_position}, " + f"width {search_map_params.width}, " + f"atlas pixel size {dcg.atlas_pixel_size}" ) murfey_db.add(search_map) murfey_db.commit() @@ -327,16 +332,16 @@ def register_batch_position_in_database( search_map.height / 2, ] tilt_series.x_location = ( - centre_batch_pixel[0] - - BatchPositionParameters.x_beamshift / search_map.pixel_size + centre_batch_pixel[0] - batch_parameters.x_beamshift / search_map.pixel_size ) tilt_series.y_location = ( - centre_batch_pixel[1] - - BatchPositionParameters.y_beamshift / search_map.pixel_size + centre_batch_pixel[1] - batch_parameters.y_beamshift / search_map.pixel_size ) else: logger.warning( - f"No search map information available to register position of {sanitise(batch_name)}" + f"Incomplete search map for position of {sanitise(batch_name)}: " + f"stage {search_map.x_stage_position}, " + f"width {search_map.width}, " ) murfey_db.add(tilt_series) murfey_db.commit() From 7c077ff6a24926770f7178a8759243ed8ec3893e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 1 Jul 2025 09:26:21 +0100 Subject: [PATCH 18/32] File pickup fix --- src/murfey/client/analyser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index fde420908..8ff32beab 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -229,7 +229,7 @@ def _analyse(self): self._context = SPAMetadataContext("epu", self._basepath) elif ( "Batch" in transferred_file.parts - or "SearchMap" in transferred_file.parts + or "SearchMaps" in transferred_file.parts or transferred_file.name == "Session.dm" and not self._context ): From 51e771d5b5e326dceec6b989e79cff285f61e55a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 1 Jul 2025 09:28:46 +0100 Subject: [PATCH 19/32] Not using atlas binning at the moment --- src/murfey/client/contexts/tomo_metadata.py | 6 ------ src/murfey/server/api/workflow.py | 6 ------ src/murfey/util/db.py | 1 - src/murfey/util/tomo_metadata.py | 1 - 4 files changed, 14 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 0318c76ac..38f2fed06 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -84,11 +84,6 @@ def post_transfer( "numericValue" ] ) - atlas_binning = int( - atlas_xml_data["MicroscopeImage"]["microscopeData"]["acquisition"][ - "camera" - ]["Binning"]["a:x"] - ) for p in partial_path.split("/"): if p.startswith("Sample"): @@ -115,7 +110,6 @@ def post_transfer( ), "sample": environment.samples[source].sample, "atlas_pixel_size": atlas_pixel_size, - "atlas_binning": atlas_binning, } capture_post(url, json=dcg_data) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 2976284fa..a36034b4a 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -84,7 +84,6 @@ class DCGroupParameters(BaseModel): atlas_pixel_size: float = 0 atlas_size_x: int = 0 atlas_size_y: int = 0 - atlas_binning: int = 1 @router.post("/visits/{visit_name}/{session_id}/register_data_collection_group") @@ -108,9 +107,6 @@ def register_dc_group( dcg_murfey[0].atlas_pixel_size = ( dcg_params.atlas_pixel_size or dcg_murfey[0].atlas_pixel_size ) - dcg_murfey[0].atlas_binning = ( - dcg_params.atlas_binning or dcg_murfey[0].atlas_binning - ) if _transport_object: if dcg_murfey[0].atlas_id is not None: @@ -124,7 +120,6 @@ def register_dc_group( "atlas_pixel_size": dcg_params.atlas_pixel_size, "atlas_size_x": dcg_params.atlas_size_x, "atlas_size_y": dcg_params.atlas_size_y, - "atlas_binning": dcg_params.atlas_binning, "dcgid": dcg_murfey[0].id, "session_id": session_id, }, @@ -151,7 +146,6 @@ def register_dc_group( "atlas": dcg_params.atlas, "sample": dcg_params.sample, "atlas_pixel_size": dcg_params.atlas_pixel_size, - "atlas_binning": dcg_params.atlas_binning, } if _transport_object: diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8cf39e6ab..b1247d2ca 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -380,7 +380,6 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore atlas_id: Optional[int] = None atlas_pixel_size: Optional[float] = None atlas: str = "" - atlas_binning: Optional[int] = None sample: Optional[int] = None session: Optional[Session] = Relationship(back_populates="data_collection_groups") data_collections: List["DataCollection"] = Relationship( diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/util/tomo_metadata.py index e07c15339..d59cdc55f 100644 --- a/src/murfey/util/tomo_metadata.py +++ b/src/murfey/util/tomo_metadata.py @@ -158,7 +158,6 @@ def register_search_map_in_database( search_map.height, search_map.width, dcg.atlas_pixel_size, - dcg.atlas_binning, ] ): reference_shift_matrix = np.array( From 8c0835a6c3f7f7e5b7213db2214a9a7601c1e381 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 1 Jul 2025 11:13:52 +0100 Subject: [PATCH 20/32] Register search maps after data collection updates --- src/murfey/server/api/session_control.py | 8 ++++---- src/murfey/server/api/workflow.py | 17 ++++++++++++++++- .../{util => workflows/tomo}/tomo_metadata.py | 0 3 files changed, 20 insertions(+), 5 deletions(-) rename src/murfey/{util => workflows/tomo}/tomo_metadata.py (100%) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 5ef15c140..c5c861f07 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -54,16 +54,16 @@ SearchMapParameters, Visit, ) -from murfey.util.tomo_metadata import ( - register_batch_position_in_database, - register_search_map_in_database, -) from murfey.workflows.spa.flush_spa_preprocess import ( register_foil_hole as _register_foil_hole, ) from murfey.workflows.spa.flush_spa_preprocess import ( register_grid_square as _register_grid_square, ) +from murfey.workflows.tomo.tomo_metadata import ( + register_batch_position_in_database, + register_search_map_in_database, +) logger = getLogger("murfey.server.api.session_control") diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index a36034b4a..feff7d6b9 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -50,6 +50,7 @@ Movie, PreprocessStash, ProcessingJob, + SearchMap, Session, SessionProcessingParameters, SPAFeedbackParameters, @@ -57,13 +58,18 @@ Tilt, TiltSeries, ) -from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo +from murfey.util.models import ( + ProcessingParametersSPA, + ProcessingParametersTomo, + SearchMapParameters, +) from murfey.util.processing_params import ( cryolo_model_path, default_spa_parameters, motion_corrected_mrc, ) from murfey.util.tomo import midpoint +from murfey.workflows.tomo.tomo_metadata import register_search_map_in_database logger = getLogger("murfey.server.api.workflow") @@ -136,6 +142,15 @@ def register_dc_group( dcg_murfey[0].atlas_id = atlas_id_response["return_value"] db.add(dcg_murfey[0]) db.commit() + + search_maps = db.exec( + select(SearchMap) + .where(SearchMap.session_id == session_id) + .where(SearchMap.tag == dcg_params.tag) + ).all() + search_map_params = SearchMapParameters(tag=dcg_params.tag) + for sm in search_maps: + register_search_map_in_database(session_id, sm.name, search_map_params, db) else: dcg_parameters = { "start_time": str(datetime.now()), diff --git a/src/murfey/util/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py similarity index 100% rename from src/murfey/util/tomo_metadata.py rename to src/murfey/workflows/tomo/tomo_metadata.py From 07d068f9250381415a693b99cd86e99880d43d87 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 1 Jul 2025 13:47:17 +0100 Subject: [PATCH 21/32] Database closure issues --- src/murfey/server/api/workflow.py | 5 ++++- src/murfey/workflows/tomo/tomo_metadata.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index feff7d6b9..64355225b 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -150,7 +150,10 @@ def register_dc_group( ).all() search_map_params = SearchMapParameters(tag=dcg_params.tag) for sm in search_maps: - register_search_map_in_database(session_id, sm.name, search_map_params, db) + register_search_map_in_database( + session_id, sm.name, search_map_params, db, close_db=False + ) + db.close() else: dcg_parameters = { "start_time": str(datetime.now()), diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index d59cdc55f..d21f3721f 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -21,6 +21,7 @@ def register_search_map_in_database( search_map_name: str, search_map_params: SearchMapParameters, murfey_db: Session, + close_db: bool = True, ): dcg = murfey_db.exec( select(DataCollectionGroup) @@ -225,7 +226,8 @@ def register_search_map_in_database( ) murfey_db.add(search_map) murfey_db.commit() - murfey_db.close() + if close_db: + murfey_db.close() def register_batch_position_in_database( From 6eb9a68392b9a1737f0bf1c889aa539495e605b1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 2 Jul 2025 15:19:19 +0100 Subject: [PATCH 22/32] Fix it for k3 flipx --- src/murfey/workflows/tomo/tomo_metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index d21f3721f..f2d231ffb 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -196,10 +196,10 @@ def register_search_map_in_database( ) camera = getattr(Camera, machine_config.camera) - if camera == Camera.FALCON or Camera.K3_FLIPY: + if camera == Camera.K3_FLIPY: # Camera.FALCON ? corrected_vector = np.matmul(np.array([[1, 0], [0, -1]]), corrected_vector) elif camera == Camera.K3_FLIPX: - corrected_vector = np.matmul(np.array([[-1, 0], [0, 1]]), corrected_vector) + corrected_vector = -1 * corrected_vector search_map_params.height_on_atlas = int( search_map.height * search_map.pixel_size / dcg.atlas_pixel_size @@ -247,6 +247,7 @@ def register_batch_position_in_database( tilt_series = murfey_db.exec( select(TiltSeries) .where(TiltSeries.tag == batch_name) + .where(TiltSeries.rsync_source == batch_parameters.tag) .where(TiltSeries.session_id == session_id) ).one() if tilt_series.x_location: From d2e6a24829e770de39d5a04c6d6998a71a2a3d0f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 2 Jul 2025 16:28:06 +0100 Subject: [PATCH 23/32] Catches for single batch position --- src/murfey/client/contexts/tomo_metadata.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 38f2fed06..172bd659d 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -235,9 +235,14 @@ def post_transfer( for_parsing = xml.read() batch_xml = xmltodict.parse(for_parsing) - for batch_position in batch_xml["BatchPositionsList"]["BatchPositions"][ + batch_positions_list = batch_xml["BatchPositionsList"]["BatchPositions"][ "BatchPositionParameters" - ]: + ] + if isinstance(batch_positions_list, dict): + # Case of a single batch + batch_positions_list = [batch_positions_list] + + for batch_position in batch_positions_list: batch_name = batch_position["Name"] search_map_name = batch_position["PositionOnTileSet"]["TileSetName"] batch_stage_location_x = float( @@ -271,7 +276,7 @@ def post_transfer( ) # Beamshifts - if batch_position["AdditionalExposureTemplateAreas"]: + if batch_position.get("AdditionalExposureTemplateAreas"): beamshifts = batch_position["AdditionalExposureTemplateAreas"][ "ExposureTemplateAreaParameters" ] From 893785e3069fce79c9d06aab21ea9d9043b5c222 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 2 Jul 2025 16:28:29 +0100 Subject: [PATCH 24/32] Flip for falcon camera --- src/murfey/workflows/tomo/tomo_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index f2d231ffb..c48815ca1 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -196,9 +196,9 @@ def register_search_map_in_database( ) camera = getattr(Camera, machine_config.camera) - if camera == Camera.K3_FLIPY: # Camera.FALCON ? + if camera == Camera.K3_FLIPY: corrected_vector = np.matmul(np.array([[1, 0], [0, -1]]), corrected_vector) - elif camera == Camera.K3_FLIPX: + elif camera == Camera.K3_FLIPX or Camera.FALCON: corrected_vector = -1 * corrected_vector search_map_params.height_on_atlas = int( From 76b0497b163f8fddce15318f580f5270ee3c4af9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 3 Jul 2025 11:12:53 +0100 Subject: [PATCH 25/32] Some tests of the search map function --- src/murfey/workflows/tomo/tomo_metadata.py | 9 +- tests/workflows/tomo/test_tomo_metadata.py | 203 +++++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tests/workflows/tomo/test_tomo_metadata.py diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index c48815ca1..7d2050e62 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -29,6 +29,7 @@ def register_search_map_in_database( .where(DataCollectionGroup.tag == search_map_params.tag) ).one() try: + # See if there is already a search map with this name and update if so search_map = murfey_db.exec( select(SearchMap) .where(SearchMap.name == search_map_name) @@ -105,6 +106,7 @@ def register_search_map_in_database( else: # mock up response so that below still works sm_ispyb_response = {"success": False, "return_value": None} + # Register new search map search_map = SearchMap( id=( sm_ispyb_response["return_value"] @@ -161,6 +163,7 @@ def register_search_map_in_database( dcg.atlas_pixel_size, ] ): + # Work out the shifted positions if all required information is present reference_shift_matrix = np.array( [ [ @@ -195,12 +198,14 @@ def register_search_map_in_database( ), ) + # Flip positions based on camera type camera = getattr(Camera, machine_config.camera) if camera == Camera.K3_FLIPY: corrected_vector = np.matmul(np.array([[1, 0], [0, -1]]), corrected_vector) - elif camera == Camera.K3_FLIPX or Camera.FALCON: - corrected_vector = -1 * corrected_vector + elif camera == Camera.K3_FLIPX: + corrected_vector = np.matmul(np.array([[-1, 0], [0, 1]]), corrected_vector) + # Convert from metres to pixels search_map_params.height_on_atlas = int( search_map.height * search_map.pixel_size / dcg.atlas_pixel_size ) diff --git a/tests/workflows/tomo/test_tomo_metadata.py b/tests/workflows/tomo/test_tomo_metadata.py new file mode 100644 index 000000000..31d7f3acb --- /dev/null +++ b/tests/workflows/tomo/test_tomo_metadata.py @@ -0,0 +1,203 @@ +from unittest import mock + +from sqlmodel import Session, select + +from murfey.util.db import DataCollectionGroup, SearchMap +from murfey.util.models import SearchMapParameters +from murfey.workflows.tomo import tomo_metadata +from tests.conftest import ExampleVisit + + +@mock.patch("murfey.workflows.tomo.tomo_metadata._transport_object") +def test_register_search_map_update_with_dimensions( + mock_transport, murfey_db_session: Session +): + """Test the updating of an existing grid square""" + # Create a grid square to update + search_map = SearchMap( + id=1, + name="SearchMap_1", + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + x_stage_position=0.1, + y_stage_position=0.2, + ) + murfey_db_session.add(search_map) + murfey_db_session.commit() + + # Make sure DCG is present with a pixel size + dcg = DataCollectionGroup( + id=1, + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + ) + murfey_db_session.add(dcg) + murfey_db_session.commit() + + # Parameters to update with + new_parameters = SearchMapParameters( + tag="session_tag", + width=2000, + height=4000, + ) + + # Run the registration + tomo_metadata.register_search_map_in_database( + ExampleVisit.murfey_session_id, "SearchMap_1", new_parameters, murfey_db_session + ) + + # Check this would have updated ispyb + mock_transport.do_update_search_map.assert_called_with(1, new_parameters) + + # Confirm the database was updated + sm_final_parameters = murfey_db_session.exec(select(SearchMap)).one() + assert sm_final_parameters.width == new_parameters.width + assert sm_final_parameters.height == new_parameters.height + assert sm_final_parameters.x_stage_position == 0.1 + assert sm_final_parameters.y_stage_position == 0.2 + assert sm_final_parameters.x_location is None + + +@mock.patch("murfey.workflows.tomo.tomo_metadata._transport_object") +def test_register_search_map_update_with_all_parameters( + mock_transport, murfey_db_session: Session +): + """Test the updating of an existing grid square""" + # Create a grid square to update + search_map = SearchMap( + id=1, + name="SearchMap_1", + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + x_stage_position=0.1, + y_stage_position=0.2, + width=2000, + height=4000, + ) + murfey_db_session.add(search_map) + murfey_db_session.commit() + + # Make sure DCG is present with a pixel size + dcg = DataCollectionGroup( + id=1, + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + ) + murfey_db_session.add(dcg) + murfey_db_session.commit() + + # Parameters to update with + new_parameters = SearchMapParameters( + tag="session_tag", + x_stage_position=0.3, + y_stage_position=0.4, + pixel_size=1e-7, + image="path/to/image", + binning=1, + reference_matrix_m11=1.01, + reference_matrix_m12=0.01, + reference_matrix_m21=0.02, + reference_matrix_m22=1.02, + stage_correction_m11=0.99, + stage_correction_m12=-0.01, + stage_correction_m21=-0.02, + stage_correction_m22=0.98, + image_shift_correction_m11=1.03, + image_shift_correction_m12=0.03, + image_shift_correction_m21=-0.03, + image_shift_correction_m22=0.97, + ) + + # Run the registration + tomo_metadata.register_search_map_in_database( + ExampleVisit.murfey_session_id, "SearchMap_1", new_parameters, murfey_db_session + ) + + # Confirm the database was updated + sm_final_parameters = murfey_db_session.exec(select(SearchMap)).one() + assert sm_final_parameters.width == new_parameters.width + assert sm_final_parameters.height == new_parameters.height + assert sm_final_parameters.x_stage_position == 0.3 + assert sm_final_parameters.y_stage_position == 0.4 + assert sm_final_parameters.pixel_size == 0.1 + assert sm_final_parameters.image == "path/to/image" + assert sm_final_parameters.binning == 1 + assert sm_final_parameters.reference_matrix_m11 == 1.01 + assert sm_final_parameters.reference_matrix_m12 == 0.01 + assert sm_final_parameters.reference_matrix_m21 == 0.02 + assert sm_final_parameters.reference_matrix_m22 == 1.02 + assert sm_final_parameters.stage_correction_m11 == 0.99 + assert sm_final_parameters.stage_correction_m12 == -0.01 + assert sm_final_parameters.stage_correction_m21 == -0.02 + assert sm_final_parameters.stage_correction_m22 == 0.98 + assert sm_final_parameters.image_shift_correction_m11 == 1.03 + assert sm_final_parameters.image_shift_correction_m12 == 0.03 + assert sm_final_parameters.image_shift_correction_m21 == -0.03 + assert sm_final_parameters.image_shift_correction_m22 == 0.97 + + # These two should have been updated, but what that update should be is messy + assert sm_final_parameters.x_location is not None + assert sm_final_parameters.y_location is not None + + # Check this would have updated ispyb + mock_transport.do_update_search_map.assert_called_with(1, new_parameters) + new_parameters.x_location = sm_final_parameters.x_location + new_parameters.y_location = sm_final_parameters.y_location + new_parameters.height_on_atlas = 40 + new_parameters.width_on_atlas = 20 + mock_transport.do_update_search_map.assert_called_with(1, new_parameters) + + +@mock.patch("murfey.workflows.tomo.tomo_metadata._transport_object") +def test_register_search_map_insert_with_ispyb( + mock_transport, murfey_db_session: Session, tmp_path +): + # Create a data collection group for lookups + dcg = DataCollectionGroup( + id=1, + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + ) + murfey_db_session.add(dcg) + murfey_db_session.commit() + + # Set the ispyb return + mock_transport.do_insert_search_map.return_value = { + "return_value": 1, + "success": True, + } + + # Parameters to update with + new_parameters = SearchMapParameters( + tag="session_tag", + x_stage_position=1.3, + y_stage_position=1.4, + pixel_size=1.02, + ) + + # Run the registration + tomo_metadata.register_search_map_in_database( + ExampleVisit.murfey_session_id, "SearchMap_1", new_parameters, murfey_db_session + ) + + # Check this would have updated ispyb + mock_transport.do_insert_search_map.assert_called_with( + 90, "SearchMap_1", new_parameters + ) + + # Confirm the database entry was made + sm_final_parameters = murfey_db_session.exec(select(SearchMap)).one() + assert sm_final_parameters.id == 1 + assert sm_final_parameters.name == "SearchMap_1" + assert sm_final_parameters.session_id == ExampleVisit.murfey_session_id + assert sm_final_parameters.tag == "session_tag" + assert sm_final_parameters.x_stage_position == 1.3 + assert sm_final_parameters.y_stage_position == 1.4 + assert sm_final_parameters.pixel_size == 1.02 + assert sm_final_parameters.x_location is None From 6ed6ce5461f50550a25a96a06ae3479b19ac4695 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 3 Jul 2025 11:23:05 +0100 Subject: [PATCH 26/32] Try and fix tests --- tests/workflows/tomo/test_tomo_metadata.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/workflows/tomo/test_tomo_metadata.py b/tests/workflows/tomo/test_tomo_metadata.py index 31d7f3acb..22fd30d2f 100644 --- a/tests/workflows/tomo/test_tomo_metadata.py +++ b/tests/workflows/tomo/test_tomo_metadata.py @@ -119,8 +119,8 @@ def test_register_search_map_update_with_all_parameters( # Confirm the database was updated sm_final_parameters = murfey_db_session.exec(select(SearchMap)).one() - assert sm_final_parameters.width == new_parameters.width - assert sm_final_parameters.height == new_parameters.height + assert sm_final_parameters.width == 2000 + assert sm_final_parameters.height == 4000 assert sm_final_parameters.x_stage_position == 0.3 assert sm_final_parameters.y_stage_position == 0.4 assert sm_final_parameters.pixel_size == 0.1 @@ -187,9 +187,7 @@ def test_register_search_map_insert_with_ispyb( ) # Check this would have updated ispyb - mock_transport.do_insert_search_map.assert_called_with( - 90, "SearchMap_1", new_parameters - ) + mock_transport.do_insert_search_map.assert_called_with(90, new_parameters) # Confirm the database entry was made sm_final_parameters = murfey_db_session.exec(select(SearchMap)).one() From 09b5f5f087b8b8c6713626e6e6d6761c87546a93 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 3 Jul 2025 12:01:29 +0100 Subject: [PATCH 27/32] Try and fix tests --- tests/workflows/tomo/test_tomo_metadata.py | 187 ++++++++++++++++++++- 1 file changed, 180 insertions(+), 7 deletions(-) diff --git a/tests/workflows/tomo/test_tomo_metadata.py b/tests/workflows/tomo/test_tomo_metadata.py index 22fd30d2f..46c534a6e 100644 --- a/tests/workflows/tomo/test_tomo_metadata.py +++ b/tests/workflows/tomo/test_tomo_metadata.py @@ -2,8 +2,8 @@ from sqlmodel import Session, select -from murfey.util.db import DataCollectionGroup, SearchMap -from murfey.util.models import SearchMapParameters +from murfey.util.db import DataCollectionGroup, SearchMap, TiltSeries +from murfey.util.models import BatchPositionParameters, SearchMapParameters from murfey.workflows.tomo import tomo_metadata from tests.conftest import ExampleVisit @@ -12,8 +12,8 @@ def test_register_search_map_update_with_dimensions( mock_transport, murfey_db_session: Session ): - """Test the updating of an existing grid square""" - # Create a grid square to update + """Test the updating of an existing search map, without enough to find location""" + # Create a search map to update search_map = SearchMap( id=1, name="SearchMap_1", @@ -64,8 +64,8 @@ def test_register_search_map_update_with_dimensions( def test_register_search_map_update_with_all_parameters( mock_transport, murfey_db_session: Session ): - """Test the updating of an existing grid square""" - # Create a grid square to update + """Test the updating of an existing search map with all required parameters""" + # Create a search map to update search_map = SearchMap( id=1, name="SearchMap_1", @@ -123,7 +123,7 @@ def test_register_search_map_update_with_all_parameters( assert sm_final_parameters.height == 4000 assert sm_final_parameters.x_stage_position == 0.3 assert sm_final_parameters.y_stage_position == 0.4 - assert sm_final_parameters.pixel_size == 0.1 + assert sm_final_parameters.pixel_size == 1e-7 assert sm_final_parameters.image == "path/to/image" assert sm_final_parameters.binning == 1 assert sm_final_parameters.reference_matrix_m11 == 1.01 @@ -156,6 +156,7 @@ def test_register_search_map_update_with_all_parameters( def test_register_search_map_insert_with_ispyb( mock_transport, murfey_db_session: Session, tmp_path ): + """Insert a new search map""" # Create a data collection group for lookups dcg = DataCollectionGroup( id=1, @@ -199,3 +200,175 @@ def test_register_search_map_insert_with_ispyb( assert sm_final_parameters.y_stage_position == 1.4 assert sm_final_parameters.pixel_size == 1.02 assert sm_final_parameters.x_location is None + + +def test_register_batch_position_update(murfey_db_session: Session): + """Test the updating of an existing tilt series with batch positions""" + # Create a tilt series to update + tilt_series = TiltSeries( + tag="Position_1", + rsync_source="session_tag", + session_id=ExampleVisit.murfey_session_id, + search_map_id=1, + ) + murfey_db_session.add(tilt_series) + murfey_db_session.commit() + + # Make sure search map is present + search_map = SearchMap( + id=1, + name="SearchMap_1", + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + x_stage_position=1, + y_stage_position=2, + pixel_size=0.01, + reference_matrix_m11=1, + reference_matrix_m12=0, + reference_matrix_m21=0, + reference_matrix_m22=1, + stage_correction_m11=1, + stage_correction_m12=0, + stage_correction_m21=0, + stage_correction_m22=1, + image_shift_correction_m11=1, + image_shift_correction_m12=0, + image_shift_correction_m21=0, + image_shift_correction_m22=1, + height=4000, + width=2000, + ) + murfey_db_session.add(search_map) + murfey_db_session.commit() + + # Parameters to update with + new_parameters = BatchPositionParameters( + tag="session_tag", + x_stage_position=0.1, + y_stage_position=0.2, + x_beamshift=0.3, + y_beamshift=0.4, + search_map_name="SearchMap_1", + ) + + # Run the registration + tomo_metadata.register_batch_position_in_database( + ExampleVisit.murfey_session_id, "Position_1", new_parameters, murfey_db_session + ) + + # These two should have been updated, values are known as used identity matrices + bp_final_parameters = murfey_db_session.exec(select(TiltSeries)).one() + assert bp_final_parameters.x_location == 880 + assert bp_final_parameters.y_location == 1780 + + +def test_register_batch_position_update_skip(murfey_db_session: Session): + """Test the updating of an existing batch position, skipped as already done""" + # Create a tilt series to update + tilt_series = TiltSeries( + tag="Position_1", + rsync_source="session_tag", + session_id=ExampleVisit.murfey_session_id, + search_map_id=1, + x_location=100, + y_location=200, + ) + murfey_db_session.add(tilt_series) + murfey_db_session.commit() + + # Make sure search map is present + search_map = SearchMap( + id=1, + name="SearchMap_1", + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + x_stage_position=1, + y_stage_position=2, + pixel_size=0.01, + reference_matrix_m11=1, + reference_matrix_m12=0, + reference_matrix_m21=0, + reference_matrix_m22=1, + stage_correction_m11=1, + stage_correction_m12=0, + stage_correction_m21=0, + stage_correction_m22=1, + image_shift_correction_m11=1, + image_shift_correction_m12=0, + image_shift_correction_m21=0, + image_shift_correction_m22=1, + height=4000, + width=2000, + ) + murfey_db_session.add(search_map) + murfey_db_session.commit() + + # Parameters to update with + new_parameters = BatchPositionParameters( + tag="session_tag", + x_stage_position=0.1, + y_stage_position=0.2, + x_beamshift=0.3, + y_beamshift=0.4, + search_map_name="SearchMap_1", + ) + + # Run the registration + tomo_metadata.register_batch_position_in_database( + ExampleVisit.murfey_session_id, "Position_1", new_parameters, murfey_db_session + ) + + # These two should have been updated, values are known as used identity matrices + bp_final_parameters = murfey_db_session.exec(select(TiltSeries)).one() + assert bp_final_parameters.x_location == 100 + assert bp_final_parameters.y_location == 200 + + +def test_register_batch_position_new(murfey_db_session: Session): + """Test the registration of a new tilt series with batch positions""" + # Make sure search map is present + search_map = SearchMap( + id=1, + name="SearchMap_1", + session_id=ExampleVisit.murfey_session_id, + tag="session_tag", + x_stage_position=1, + y_stage_position=2, + pixel_size=0.01, + reference_matrix_m11=1, + reference_matrix_m12=0, + reference_matrix_m21=0, + reference_matrix_m22=1, + stage_correction_m11=1, + stage_correction_m12=0, + stage_correction_m21=0, + stage_correction_m22=1, + image_shift_correction_m11=1, + image_shift_correction_m12=0, + image_shift_correction_m21=0, + image_shift_correction_m22=1, + height=4000, + width=2000, + ) + murfey_db_session.add(search_map) + murfey_db_session.commit() + + # Parameters to update with + new_parameters = BatchPositionParameters( + tag="session_tag", + x_stage_position=0.1, + y_stage_position=0.2, + x_beamshift=0.3, + y_beamshift=0.4, + search_map_name="SearchMap_1", + ) + + # Run the registration + tomo_metadata.register_batch_position_in_database( + ExampleVisit.murfey_session_id, "Position_1", new_parameters, murfey_db_session + ) + + # These two should have been updated, values are known as used identity matrices + bp_final_parameters = murfey_db_session.exec(select(TiltSeries)).one() + assert bp_final_parameters.x_location == 880 + assert bp_final_parameters.y_location == 1780 From bcb8b8b989b6e820858beb64ba5b7595e8058665 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 3 Jul 2025 12:08:18 +0100 Subject: [PATCH 28/32] Try and fix tests --- tests/workflows/tomo/test_tomo_metadata.py | 59 +++++++++------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/tests/workflows/tomo/test_tomo_metadata.py b/tests/workflows/tomo/test_tomo_metadata.py index 46c534a6e..9febae4d4 100644 --- a/tests/workflows/tomo/test_tomo_metadata.py +++ b/tests/workflows/tomo/test_tomo_metadata.py @@ -98,18 +98,9 @@ def test_register_search_map_update_with_all_parameters( pixel_size=1e-7, image="path/to/image", binning=1, - reference_matrix_m11=1.01, - reference_matrix_m12=0.01, - reference_matrix_m21=0.02, - reference_matrix_m22=1.02, - stage_correction_m11=0.99, - stage_correction_m12=-0.01, - stage_correction_m21=-0.02, - stage_correction_m22=0.98, - image_shift_correction_m11=1.03, - image_shift_correction_m12=0.03, - image_shift_correction_m21=-0.03, - image_shift_correction_m22=0.97, + reference_matrix={"m11": 1.01, "m12": 0.01, "m21": 0.02, "m22": 1.02}, + stage_correction={"m11": 0.99, "m12": -0.01, "m21": -0.02, "m22": 0.98}, + image_shift_correction={"m11": 1.03, "m12": 0.03, "m21": -0.03, "m22": 0.97}, ) # Run the registration @@ -204,16 +195,6 @@ def test_register_search_map_insert_with_ispyb( def test_register_batch_position_update(murfey_db_session: Session): """Test the updating of an existing tilt series with batch positions""" - # Create a tilt series to update - tilt_series = TiltSeries( - tag="Position_1", - rsync_source="session_tag", - session_id=ExampleVisit.murfey_session_id, - search_map_id=1, - ) - murfey_db_session.add(tilt_series) - murfey_db_session.commit() - # Make sure search map is present search_map = SearchMap( id=1, @@ -241,6 +222,16 @@ def test_register_batch_position_update(murfey_db_session: Session): murfey_db_session.add(search_map) murfey_db_session.commit() + # Create a tilt series to update + tilt_series = TiltSeries( + tag="Position_1", + rsync_source="session_tag", + session_id=ExampleVisit.murfey_session_id, + search_map_id=1, + ) + murfey_db_session.add(tilt_series) + murfey_db_session.commit() + # Parameters to update with new_parameters = BatchPositionParameters( tag="session_tag", @@ -264,18 +255,6 @@ def test_register_batch_position_update(murfey_db_session: Session): def test_register_batch_position_update_skip(murfey_db_session: Session): """Test the updating of an existing batch position, skipped as already done""" - # Create a tilt series to update - tilt_series = TiltSeries( - tag="Position_1", - rsync_source="session_tag", - session_id=ExampleVisit.murfey_session_id, - search_map_id=1, - x_location=100, - y_location=200, - ) - murfey_db_session.add(tilt_series) - murfey_db_session.commit() - # Make sure search map is present search_map = SearchMap( id=1, @@ -303,6 +282,18 @@ def test_register_batch_position_update_skip(murfey_db_session: Session): murfey_db_session.add(search_map) murfey_db_session.commit() + # Create a tilt series to update + tilt_series = TiltSeries( + tag="Position_1", + rsync_source="session_tag", + session_id=ExampleVisit.murfey_session_id, + search_map_id=1, + x_location=100, + y_location=200, + ) + murfey_db_session.add(tilt_series) + murfey_db_session.commit() + # Parameters to update with new_parameters = BatchPositionParameters( tag="session_tag", From 8301d7e294c2c6c0c3ee2ef85f2c637d2f365ade Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 3 Jul 2025 12:12:25 +0100 Subject: [PATCH 29/32] Fix codeql issue --- src/murfey/workflows/tomo/tomo_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index 7d2050e62..02c1c60c9 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -225,9 +225,9 @@ def register_search_map_in_database( else: logger.info( f"Unable to register search map {sanitise(search_map_name)} position yet: " - f"stage {search_map_params.x_stage_position}, " - f"width {search_map_params.width}, " - f"atlas pixel size {dcg.atlas_pixel_size}" + f"stage {sanitise(str(search_map_params.x_stage_position))}, " + f"width {sanitise(str(search_map_params.width))}, " + f"atlas pixel size {sanitise(str(dcg.atlas_pixel_size))}" ) murfey_db.add(search_map) murfey_db.commit() From 14ee131860eabd59a88709d4f184635a5caf728e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 7 Jul 2025 16:42:03 +0100 Subject: [PATCH 30/32] If no size is present for a search map, just have to insert something reasonable enough --- src/murfey/client/contexts/tomo_metadata.py | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 172bd659d..9945db886 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -215,8 +215,27 @@ def post_transfer( sm_data = xmltodict.parse(sm_xml.read()) # This bit gets SearchMap size - sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) - sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) + try: + sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"]) + sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"]) + except KeyError: + logger.warning(f"Unable to find size for SearchMap {transferred_file}") + readout_width = int( + sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][ + "ReadoutArea" + ]["b:width"] + ) + readout_height = int( + sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][ + "ReadoutArea" + ]["b:height"] + ) + sm_width = int( + 8005 * readout_width / max(readout_height, readout_width) + ) + sm_height = int( + 8005 * readout_height / max(readout_height, readout_width) + ) sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( From 4393ebb64da44fa18c59f4d67fe55460f6fc9506 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 8 Jul 2025 09:24:21 +0100 Subject: [PATCH 31/32] Make clear that inserted values are wrong --- src/murfey/client/contexts/tomo_metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 9945db886..07b154acc 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -236,6 +236,9 @@ def post_transfer( sm_height = int( 8005 * readout_height / max(readout_height, readout_width) ) + logger.warning( + f"Inserting incorrect width {sm_width}, height {sm_height} for SearchMap display" + ) sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}" capture_post( From 538ef47c166c79135be4659511f0c9c3cfd0e1cc Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 8 Jul 2025 10:33:44 +0100 Subject: [PATCH 32/32] Fixes --- src/murfey/client/contexts/tomo_metadata.py | 4 ++-- src/murfey/util/db.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 07b154acc..a93a7b644 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -222,12 +222,12 @@ def post_transfer( logger.warning(f"Unable to find size for SearchMap {transferred_file}") readout_width = int( sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][ - "ReadoutArea" + "a:ReadoutArea" ]["b:width"] ) readout_height = int( sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][ - "ReadoutArea" + "a:ReadoutArea" ]["b:height"] ) sm_width = int( diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index b1247d2ca..8dee78fcf 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -353,7 +353,10 @@ class TiltSeries(SQLModel, table=True): # type: ignore tag: str rsync_source: str session_id: int = Field(foreign_key="session.id") - search_map_id: int = Field(foreign_key="searchmap.id") + search_map_id: Optional[int] = Field( + foreign_key="searchmap.id", + default=None, + ) tilt_series_length: int = -1 processing_requested: bool = False x_location: Optional[float] = None