diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 40e8d9068..7043598f1 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -24,7 +24,7 @@ from murfey.client.tui.forms import FormDependency from murfey.util.client import Observer, get_machine_config_client from murfey.util.mdoc import get_block -from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA +from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo logger = logging.getLogger("murfey.client.analyser") @@ -65,7 +65,7 @@ def __init__( self._environment = environment self._force_mdoc_metadata = force_mdoc_metadata self.parameters_model: ( - Type[ProcessingParametersSPA] | Type[PreprocessingParametersTomo] | None + Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None ) = None self.queue: queue.Queue = queue.Queue() @@ -183,7 +183,7 @@ def _find_context(self, file_path: Path) -> bool: if not self._context: logger.info("Acquisition software: tomo") self._context = TomographyContext("tomo", self._basepath) - self.parameters_model = PreprocessingParametersTomo + self.parameters_model = ProcessingParametersTomo return True # Files with these suffixes belong to the serial EM tomography workflow @@ -207,7 +207,7 @@ def _find_context(self, file_path: Path) -> bool: ): return False self._context = TomographyContext("serialem", self._basepath) - self.parameters_model = PreprocessingParametersTomo + self.parameters_model = ProcessingParametersTomo return True return False diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 138f3e9ef..b69d76d8b 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -23,7 +23,6 @@ get_machine_config_client, ) from murfey.util.mdoc import get_block, get_global_data, get_num_blocks -from murfey.util.tomo import midpoint logger = logging.getLogger("murfey.client.contexts.tomo") @@ -72,7 +71,6 @@ class TomographyContext(Context): ProcessingParameter( "dose_per_frame", "Dose Per Frame (e- / Angstrom^2 / frame)", default=1 ), - ProcessingParameter("manual_tilt_offset", "Tilt Offset", default=0), ProcessingParameter("gain_ref", "Gain Reference"), ProcessingParameter("eer_fractionation", "EER Fractionation", default=20), ] @@ -570,7 +568,6 @@ def gather_metadata( if environment else None ) - metadata["manual_tilt_offset"] = 0 metadata["source"] = str(self._basepath) except KeyError: return OrderedDict({}) @@ -631,9 +628,6 @@ def gather_metadata( if environment else None ) - mdoc_metadata["manual_tilt_offset"] = -midpoint( - [float(b["TiltAngle"]) for b in blocks] - ) mdoc_metadata["source"] = str(self._basepath) mdoc_metadata["tag"] = str(self._basepath) mdoc_metadata["tilt_series_tag"] = metadata_file.stem diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 4d115d23e..21f3455c0 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -410,12 +410,6 @@ def _start_dc(self, json, from_form: bool = False): source = Path(json["source"]) context = self.analysers[source]._context if isinstance(context, TomographyContext): - if from_form: - capture_post( - f"{self._environment.url.geturl()}/clients/{self._environment.client_id}/tomography_processing_parameters", - json=json, - ) - source = Path(json["source"]) context.register_tomography_data_collections( file_extension=json["file_extension"], @@ -443,7 +437,7 @@ def _start_dc(self, json, from_form: bool = False): eer_fractionation_file = eer_response.json()["eer_fractionation_file"] json.update({"eer_fractionation_file": eer_fractionation_file}) capture_post( - f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_preprocessing_parameters", + f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_processing_parameters", json=json, ) capture_post( diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index 0d5060a55..8ef652cd6 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -504,7 +504,7 @@ def _start_dc(self, json, from_form: bool = False): eer_fractionation_file = eer_response.json()["eer_fractionation_file"] json.update({"eer_fractionation_file": eer_fractionation_file}) requests.post( - f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_preprocessing_parameters", + f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_processing_parameters", json=json, ) capture_post( diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 0ff7d839b..971603d67 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -58,7 +58,7 @@ from murfey.client.tui.forms import FormDependency from murfey.util import posix_path from murfey.util.client import capture_post, get_machine_config_client, read_config -from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA +from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo log = logging.getLogger("murfey.tui.screens") @@ -464,7 +464,7 @@ def compose(self): def _write_params( self, params: dict | None = None, - model: PreprocessingParametersTomo | ProcessingParametersSPA | None = None, + model: ProcessingParametersTomo | ProcessingParametersSPA | None = None, ): if params: try: @@ -474,9 +474,9 @@ def _write_params( for k in analyser._context.user_params + analyser._context.metadata_params: self.app.query_one("#info").write(f"{k.label}: {params.get(k.name)}") self.app._start_dc(params) - if model == PreprocessingParametersTomo: + if model == ProcessingParametersTomo: requests.post( - f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_preprocessing_parameters", + f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_processing_parameters", json=params, ) elif model == ProcessingParametersSPA: @@ -1157,9 +1157,7 @@ def on_input_changed(self, event): def on_button_pressed(self, event): if self.app._multigrid and self.app._processing_enabled: if self._context == TomographyContext: - valid = validate_form( - self._user_params, PreprocessingParametersTomo.Base - ) + valid = validate_form(self._user_params, ProcessingParametersTomo.Base) else: valid = validate_form(self._user_params, ProcessingParametersSPA.Base) if not valid: diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 94a5431bb..92dfc44b3 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -189,19 +189,10 @@ def get_job_ids(tilt_series_id: int, appid: int) -> JobIDs: ) -def get_tomo_proc_params(pj_id: int, *args) -> db.TomographyProcessingParameters: +def get_tomo_proc_params(dcg_id: int, *args) -> db.TomographyProcessingParameters: results = murfey_db.exec( select(db.TomographyProcessingParameters).where( - db.TomographyProcessingParameters.pj_id == pj_id - ) - ).one() - return results - - -def get_tomo_preproc_params(dcg_id: int, *args) -> db.TomographyPreprocessingParameters: - results = murfey_db.exec( - select(db.TomographyPreprocessingParameters).where( - db.TomographyPreprocessingParameters.dcg_id == dcg_id + db.TomographyProcessingParameters.dcg_id == dcg_id ) ).one() return results @@ -1660,7 +1651,7 @@ def _flush_tomography_preprocessing(message: dict): .where(db.DataCollectionGroup.session_id == session_id) .where(db.DataCollectionGroup.tag == message["data_collection_group_tag"]) ).first() - proc_params = get_tomo_preproc_params(collected_ids.id) + proc_params = get_tomo_proc_params(collected_ids.id) if not proc_params: visit_name = message["visit_name"].replace("\r\n", "").replace("\n", "") logger.warning( @@ -2160,7 +2151,7 @@ def feedback_callback(header: dict, message: dict) -> None: ] tilts = get_all_tilts(relevant_tilt_series.id) ids = get_job_ids(relevant_tilt_series.id, alignment_ids[2].id) - preproc_params = get_tomo_preproc_params(ids.dcgid) + preproc_params = get_tomo_proc_params(ids.dcgid) stack_file = ( Path(message["mrc_out"]).parents[3] / "Tomograms" @@ -2534,11 +2525,11 @@ def feedback_callback(header: dict, message: dict) -> None: .where(db.ProcessingJob.recipe == "em-tomo-preprocess") ).one() if not murfey_db.exec( - select(func.count(db.TomographyPreprocessingParameters.dcg_id)).where( - db.TomographyPreprocessingParameters.dcg_id == collected_ids[0].id + select(func.count(db.TomographyProcessingParameters.dcg_id)).where( + db.TomographyProcessingParameters.dcg_id == collected_ids[0].id ) ).one(): - params = db.TomographyPreprocessingParameters( + params = db.TomographyProcessingParameters( dcg_id=collected_ids[0].id, pixel_size=float(message["pixel_size_on_image"]) * 10**10, voltage=message["voltage"], diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index 604feffb7..08b1cd94c 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -42,7 +42,7 @@ get_job_ids, get_machine_config, get_microscope, - get_tomo_preproc_params, + get_tomo_proc_params, sanitise, templates, ) @@ -70,7 +70,6 @@ SPARelionParameters, Tilt, TiltSeries, - TomographyProcessingParameters, ) from murfey.util.models import ( BLSampleImageParameters, @@ -86,7 +85,6 @@ GridSquareParameters, MillingParameters, PostInfo, - PreprocessingParametersTomo, ProcessingJobParameters, ProcessingParametersSPA, ProcessingParametersTomo, @@ -618,9 +616,9 @@ def post_foil_hole( return register_foil_hole(session_id, gs_name, foil_hole_params, db) -@router.post("/sessions/{session_id}/tomography_preprocessing_parameters") -def register_tomo_preproc_params( - session_id: MurfeySessionID, proc_params: PreprocessingParametersTomo, db=murfey_db +@router.post("/sessions/{session_id}/tomography_processing_parameters") +def register_tomo_proc_params( + session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db ): session_processing_parameters = db.exec( select(SessionProcessingParameters).where( @@ -643,41 +641,6 @@ def register_tomo_preproc_params( _transport_object.send(_transport_object.feedback_queue, zocalo_message) -@router.post("/sessions/{session_id}/tomography_processing_parameters") -def register_tomo_proc_params( - session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db -): - log.info( - f"Registering tomography processing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}" - ) - collected_ids = db.exec( - select( - DataCollectionGroup, - DataCollection, - ProcessingJob, - AutoProcProgram, - ) - .where(DataCollectionGroup.session_id == session_id) - .where(DataCollectionGroup.tag == proc_params.tag) - .where(DataCollection.tag == proc_params.tilt_series_tag) - .where(DataCollection.dcg_id == DataCollectionGroup.id) - .where(ProcessingJob.dc_id == DataCollection.id) - .where(AutoProcProgram.pj_id == ProcessingJob.id) - .where(ProcessingJob.recipe == "em-tomo-preprocess") - ).one() - if not db.exec( - select(func.count(TomographyProcessingParameters.pj_id)).where( - TomographyProcessingParameters.pj_id == collected_ids[2].id - ) - ).one(): - tomogram_params = TomographyProcessingParameters( - pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset - ) - db.add(tomogram_params) - db.commit() - db.close() - - class Tag(BaseModel): tag: str @@ -804,7 +767,7 @@ def register_completed_tilt_series( ] tilts = get_all_tilts(ts.id) ids = get_job_ids(ts.id, collected_ids[3].id) - preproc_params = get_tomo_preproc_params(ids.dcgid) + preproc_params = get_tomo_proc_params(ids.dcgid) first_tilt = db.exec( select(Tilt).where(Tilt.tilt_series_id == ts.id) diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 58621c759..01163ec05 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -57,7 +57,6 @@ SPARelionParameters, Tilt, TiltSeries, - TomographyPreprocessingParameters, TomographyProcessingParameters, ) from murfey.util.models import ( @@ -70,7 +69,6 @@ GainReference, GridSquareParameters, PostInfo, - PreprocessingParametersTomo, ProcessingJobParameters, ProcessingParametersSPA, ProcessingParametersTomo, @@ -476,9 +474,9 @@ def register_spa_proc_params( db.commit() -@router.post("/sessions/{session_id}/tomography_preprocessing_parameters") -def register_tomo_preproc_params( - session_id: MurfeySessionID, proc_params: PreprocessingParametersTomo, db=murfey_db +@router.post("/sessions/{session_id}/tomography_processing_parameters") +def register_tomo_proc_params( + session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db ): log.info( f"Registering tomography preprocessing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}" @@ -499,68 +497,19 @@ def register_tomo_preproc_params( .where(ProcessingJob.recipe == "em-tomo-preprocess") ).one() if not db.exec( - select(func.count(TomographyPreprocessingParameters.dcg_id)).where( - TomographyPreprocessingParameters.dcg_id == collected_ids[0].id + select(func.count(TomographyProcessingParameters.dcg_id)).where( + TomographyProcessingParameters.dcg_id == collected_ids[0].id ) ).one(): - params = TomographyPreprocessingParameters( + params = TomographyProcessingParameters( dcg_id=collected_ids[0].id, pixel_size=proc_params.pixel_size_on_image, dose_per_frame=proc_params.dose_per_frame, gain_ref=proc_params.gain_ref, motion_corr_binning=proc_params.motion_corr_binning, voltage=proc_params.voltage, - # manual_tilt_offset=proc_params.manual_tilt_offset, ) db.add(params) - if not db.exec( - select(func.count(TomographyProcessingParameters.pj_id)).where( - TomographyProcessingParameters.pj_id == collected_ids[2].id - ) - ).one(): - tomogram_params = TomographyProcessingParameters( - pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset - ) - db.add(tomogram_params) - db.commit() - db.close() - - -@router.post("/clients/{client_id}/tomography_processing_parameters") -def register_tomo_proc_params( - client_id: int, proc_params: ProcessingParametersTomo, db=murfey_db -): - client = db.exec( - select(ClientEnvironment).where(ClientEnvironment.client_id == client_id) - ).one() - session_id = client.session_id - log.info( - f"Registering tomography processing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}, {session_id}" - ) - collected_ids = db.exec( - select( - DataCollectionGroup, - DataCollection, - ProcessingJob, - AutoProcProgram, - ) - .where(DataCollectionGroup.session_id == session_id) - .where(DataCollectionGroup.tag == proc_params.tag) - .where(DataCollection.tag == proc_params.tilt_series_tag) - .where(DataCollection.dcg_id == DataCollectionGroup.id) - .where(ProcessingJob.dc_id == DataCollection.id) - .where(AutoProcProgram.pj_id == ProcessingJob.id) - .where(ProcessingJob.recipe == "em-tomo-preprocess") - ).one() - if not db.exec( - select(func.count(TomographyProcessingParameters.pj_id)).where( - TomographyProcessingParameters.pj_id == collected_ids[2].id - ) - ).one(): - tomogram_params = TomographyProcessingParameters( - pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset - ) - db.add(tomogram_params) db.commit() db.close() diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index c494684d9..c5c65b456 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -378,7 +378,7 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, ) - tomography_preprocessing_parameters: List["TomographyPreprocessingParameters"] = ( + tomography_processing_parameters: List["TomographyProcessingParameters"] = ( Relationship( back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, @@ -448,12 +448,6 @@ class ProcessingJob(SQLModel, table=True): # type: ignore spa_feedback_parameters: List["SPAFeedbackParameters"] = Relationship( back_populates="processing_job", sa_relationship_kwargs={"cascade": "delete"} ) - tomography_processing_parameters: List["TomographyProcessingParameters"] = ( - Relationship( - back_populates="processing_job", - sa_relationship_kwargs={"cascade": "delete"}, - ) - ) ctf_parameters: List["CtfParameters"] = Relationship( back_populates="processing_job", sa_relationship_kwargs={"cascade": "delete"} ) @@ -499,7 +493,7 @@ class SelectionStash(SQLModel, table=True): # type: ignore ) -class TomographyPreprocessingParameters(SQLModel, table=True): # type: ignore +class TomographyProcessingParameters(SQLModel, table=True): # type: ignore dcg_id: int = Field(primary_key=True, foreign_key="datacollectiongroup.id") pixel_size: float dose_per_frame: float @@ -510,14 +504,6 @@ class TomographyPreprocessingParameters(SQLModel, table=True): # type: ignore motion_corr_binning: int = 1 gain_ref: Optional[str] = None data_collection_group: Optional[DataCollectionGroup] = Relationship( - back_populates="tomography_preprocessing_parameters" - ) - - -class TomographyProcessingParameters(SQLModel, table=True): # type: ignore - pj_id: int = Field(primary_key=True, foreign_key="processingjob.id") - manual_tilt_offset: int - processing_job: Optional[ProcessingJob] = Relationship( back_populates="tomography_processing_parameters" ) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 2da0bd345..fac7bfe0a 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -337,7 +337,7 @@ class CompletedTiltSeries(BaseModel): rsync_source: str -class PreprocessingParametersTomo(BaseModel): +class ProcessingParametersTomo(BaseModel): dose_per_frame: Optional[float] frame_count: int tilt_axis: float @@ -348,7 +348,6 @@ class PreprocessingParametersTomo(BaseModel): image_size_y: int pixel_size_on_image: str motion_corr_binning: int - manual_tilt_offset: float file_extension: str tag: str tilt_series_tag: str @@ -358,11 +357,4 @@ class PreprocessingParametersTomo(BaseModel): class Base(BaseModel): dose_per_frame: Optional[float] gain_ref: Optional[str] - manual_tilt_offset: float eer_fractionation: int - - -class ProcessingParametersTomo(BaseModel): - manual_tilt_offset: int - tag: str - tilt_series_tag: str