diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index 81f6bd83f..cacbd035d 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -35,11 +35,7 @@ def post_transfer( # These have no extensions, and end with one of the listed suffixes if not transferred_file.suffix and transferred_file.stem.endswith( ( - # Fluorescent SIM raw data files end as follows - "_BR", - "_BFR", - "_GR", - "_GFR", + # Only SIM raw data files ending with '_FL' should be processed "_BR_FL", "_BFR_FL", "_GR_FL", @@ -62,7 +58,7 @@ def post_transfer( capture_post( base_url=str(environment.url.geturl()), router_name="workflow_sim.router", - function_name="request_sim_processing", + function_name="request_sim_reconstruction", token=self._token, instrument_name=environment.instrument_name, data={ diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py index 8bc91bc07..850ef51f3 100644 --- a/src/murfey/server/api/workflow_sim.py +++ b/src/murfey/server/api/workflow_sim.py @@ -1,13 +1,18 @@ import json import logging from pathlib import Path +from typing import Any from fastapi import APIRouter, Depends from pydantic import BaseModel +from sqlmodel import Session as SQLModelSession, select +import murfey.util.db as MurfeyDB from murfey.server import _transport_object from murfey.server.api.auth import validate_instrument_token +from murfey.server.murfey_db import murfey_db from murfey.util import sanitise_path +from murfey.util.config import get_machine_config logger = logging.getLogger("murfey.server.api.workflow_sim") @@ -22,22 +27,99 @@ class SIMDataFile(BaseModel): file: Path -@router.post("/sessions/{session_id}/process_data") -def request_sim_processing(session_id: int, sim_data: SIMDataFile): +@router.post("/sessions/{session_id}/sim_recon") +def request_sim_reconstruction( + session_id: int, + sim_data: SIMDataFile, + murfey_db: SQLModelSession = murfey_db, +): if _transport_object is None: logger.error("No TransportManager object was set up") return None + # Load instrument and visit information based on session + try: + murfey_session = murfey_db.exec( + select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) + ).one() + instrument_name = murfey_session.instrument_name + visit_name = murfey_session.visit + except Exception: + logger.error("Error querying session information from database", exc_info=True) + return None + + # Load PySIMRecon values from the machine config + try: + machine_config = get_machine_config(instrument_name)[instrument_name] + pysimrecon_config: dict[str, dict[str, Any]] | None = ( + machine_config.calibrations.get("pysimrecon_config") + ) + if not pysimrecon_config: + # If no calibration was provided, use defaults + # Values provided on 2026-07-16 + pysimrecon_config = { + "blue": { + "wavelength": 452, + "ls": 0.330, + "beaddiam": 0.220, + }, + "green": { + "wavelength": 525, + "ls": 0.394, + }, + "red": { + "wavelength": 605, + "ls": 0.451, + }, + "far_red": { + "wavelength": 655, + "ls": 0.521, + }, + } + logger.warning( + f"No PySIMRecon configuration found for {instrument_name}; " + f"using known defaults \n{json.dumps(pysimrecon_config, indent=2)}" + ) + except Exception: + logger.error("Error loading machine config from database", exc_info=True) + return None + # Construct message and submit it to 'processing_recipe' logger.info( f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}" ) + # Construct the output directory for the PySIMRecon outputs to be saved to + try: + visit_idx = sim_data.file.parts.index(visit_name) + raw_dir = Path( + "/".join( + "" + if part == "/" # Replace root "/" with "" for Linux paths + else part + for part in sim_data.file.parts[: visit_idx + 2] + ) + ) + output_dir = ( + raw_dir.parent / "processed" / sim_data.file.parent.relative_to(raw_dir) + ) + except Exception: + logger.error( + "Could not determine the output directory to save the cryoSIM file " + f"{sanitise_path(sim_data.file)} to" + ) + return None recipe = { - "recipes": ["sim-process-data"], + "recipes": ["sim-reconstruction"], "parameters": { - # Job parameters - "session_id": session_id, + # PySIMRecon parameters "file": f"{str(sim_data.file)}", + "output_dir": str(output_dir), + "blue_params": str(pysimrecon_config["blue"]), + "green_params": str(pysimrecon_config["green"]), + "red_params": str(pysimrecon_config["red"]), + "far_red_params": str(pysimrecon_config["far_red"]), + # Return message + "session_id": session_id, "feedback_queue": _transport_object.feedback_queue, }, } @@ -46,6 +128,6 @@ def request_sim_processing(session_id: int, sim_data: SIMDataFile): f"{json.dumps(recipe, indent=2, default=str)}" ) # Disabled for now; will submit message once recipe and service have been set up - # _transport_object.send( - # queue="processing_recipe", message=recipe, new_connection=True - # ) + _transport_object.send( + queue="processing_recipe", message=recipe, new_connection=True + ) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 0e5799fff..b37209141 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1453,8 +1453,8 @@ murfey.server.api.workflow_fib.router: methods: - POST murfey.server.api.workflow_sim.router: - - path: /workflow/sim/sessions/{session_id}/process_data - function: request_sim_processing + - path: /workflow/sim/sessions/{session_id}/sim_recon + function: request_sim_reconstruction path_params: - name: session_id type: int diff --git a/tests/client/contexts/test_sim.py b/tests/client/contexts/test_sim.py index 80acf2d93..b9bd78911 100644 --- a/tests/client/contexts/test_sim.py +++ b/tests/client/contexts/test_sim.py @@ -29,6 +29,7 @@ def sim_data(visit_dir: Path): "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR", + # To be processed "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL", "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL", "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL", @@ -132,7 +133,7 @@ def test_post_transfer( destination_files = [ destination_dir / file.relative_to(visit_dir) for file in sim_data - if not file.stem.endswith("_BF") + if file.stem.endswith("_FL") ] # Mock the functions used in 'post_transfer' @@ -162,7 +163,7 @@ def test_post_transfer( mock_logger.warning.assert_called_with(f"No source found for file {file}") else: for src, dst in zip(sim_data, [Path(""), *destination_files]): - if src.stem.endswith("_BF"): + if not src.stem.endswith("_FL"): continue else: mock_get_source.assert_any_call(src, mock_environment) @@ -175,7 +176,7 @@ def test_post_transfer( mock_capture_post.assert_any_call( base_url=mock.ANY, router_name="workflow_sim.router", - function_name="request_sim_processing", + function_name="request_sim_reconstruction", token=context._token, instrument_name=instrument_name, data={ @@ -184,4 +185,4 @@ def test_post_transfer( # Endpoint kwargs session_id=session_id, ) - assert mock_capture_post.call_count == len(sim_data) - 1 + assert mock_capture_post.call_count == 4 diff --git a/tests/server/api/test_workflow_sim.py b/tests/server/api/test_workflow_sim.py index a17d7fd92..dc878e52d 100644 --- a/tests/server/api/test_workflow_sim.py +++ b/tests/server/api/test_workflow_sim.py @@ -5,20 +5,109 @@ import pytest from pytest_mock import MockerFixture -from murfey.server.api.workflow_sim import SIMDataFile, request_sim_processing +from murfey.server.api.workflow_sim import SIMDataFile, request_sim_reconstruction +from murfey.util import sanitise_path +from murfey.util.config import MachineConfig +# Global variables +instrument_name = "sim" +visit_name = "cm12345-6" +session_id = 1 -@pytest.mark.parametrize("has_transport_object", (True, False)) -def test_request_sim_processing( - mocker: MockerFixture, tmp_path: Path, has_transport_object: bool + +@pytest.fixture +def visit_dir(tmp_path: Path): + visit_dir = tmp_path / "data" / "2020" / visit_name + visit_dir.mkdir(parents=True, exist_ok=True) + return visit_dir + + +@pytest.mark.parametrize( + "test_params", + ( # Transport object | DB query success | Machine config found | PySIMRecon config found | Output dir found + # Successful case + (True, True, True, True, True), + (True, True, True, False, True), # No PySIMRecon config + # Failure cases + (False, True, True, True, True), # No transport object + (True, False, True, True, True), # DB query failed + (True, True, False, True, True), # Machine config error + (True, True, True, True, False), # Incorrect output dir + ), +) +def test_request_sim_reconstruction( + mocker: MockerFixture, + tmp_path: Path, + visit_dir: Path, + test_params: tuple[bool, bool, bool, bool, bool], ): - # Set up the variables - session_id = 1 - sim_data = SIMDataFile(**{"file": str(tmp_path / "dummy")}) + # Unpack test params + ( + has_transport_object, + db_query_success, + machine_config_found, + pysimrecon_configured, + output_dir_success, + ) = test_params + + # Set up the test file and output directory + test_file = ( + visit_dir / "raw" / "grid_1" / "test_file" + if output_dir_success + else tmp_path / "dummy" # Provide incorrect file path + ) + sim_data = SIMDataFile(**{"file": str(test_file)}) + output_dir = visit_dir / "processed" / "grid_1" + output_dir.mkdir(parents=True, exist_ok=True) # Mock the logger mock_logger = mocker.patch("murfey.server.api.workflow_sim.logger") + # Mock the Murfey DB + mock_murfey_session = MagicMock( + instrument_name=instrument_name, + visit=visit_name, + ) + mock_db = MagicMock() + if db_query_success: + mock_db.exec.return_value.one.return_value = mock_murfey_session + else: + mock_db.exec.return_value.one.side_effect = Exception("Something went wrong") + + # Mock the machine config + blue_params = { + "wavelength": 452, + "ls": 0.123 if pysimrecon_configured else 0.330, + "beaddiam": 0.220, + } + green_params = { + "wavelength": 525, + "ls": 0.234 if pysimrecon_configured else 0.394, + } + red_params = { + "wavelength": 605, + "ls": 0.345 if pysimrecon_configured else 0.451, + } + far_red_params = { + "wavelength": 655, + "ls": 0.456 if pysimrecon_configured else 0.521, + } + pysimrecon_config = { + "blue": blue_params, + "green": green_params, + "red": red_params, + "far_red": far_red_params, + } + machine_config = MachineConfig( + calibrations={"pysimrecon_config": pysimrecon_config} + if pysimrecon_configured + else {}, + ) + mocker.patch( + "murfey.server.api.workflow_sim.get_machine_config", + return_value={instrument_name: machine_config} if machine_config_found else {}, + ) + # Mock the transport object if has_transport_object: mock_transport_object = MagicMock() @@ -34,18 +123,41 @@ def test_request_sim_processing( ) # Run the function and check that the expected calls were made - request_sim_processing( - session_id=session_id, - sim_data=sim_data, + request_sim_reconstruction( + session_id=session_id, sim_data=sim_data, murfey_db=mock_db ) # Check that the expected calls were made - if has_transport_object: + # The parameters are toggled 'False' one at a time + if not has_transport_object: + mock_logger.error.assert_called_with("No TransportManager object was set up") + elif not db_query_success: + mock_logger.error.assert_called_with( + "Error querying session information from database", exc_info=True + ) + mock_transport_object.send.assert_not_called() + elif not machine_config_found: + mock_logger.error.assert_called_with( + "Error loading machine config from database", exc_info=True + ) + mock_transport_object.send.assert_not_called() + elif not output_dir_success: + mock_logger.error.assert_called_with( + "Could not determine the output directory to save the cryoSIM file " + f"{sanitise_path(sim_data.file)} to" + ) + mock_transport_object.send.assert_not_called() + else: recipe = { - "recipes": ["sim-process-data"], + "recipes": ["sim-reconstruction"], "parameters": { + "file": f"{str(sim_data.file)}", + "output_dir": str(output_dir), + "blue_params": str(pysimrecon_config["blue"]), + "green_params": str(pysimrecon_config["green"]), + "red_params": str(pysimrecon_config["red"]), + "far_red_params": str(pysimrecon_config["far_red"]), "session_id": session_id, - "file": f"{sim_data.file}", "feedback_queue": "dummy", }, } @@ -53,8 +165,6 @@ def test_request_sim_processing( "Will submit the following message to 'processing_recipe':\n" f"{json.dumps(recipe, indent=2, default=str)}" ) - # mock_transport_object.send.assert_called_with( - # queue="processing_recipe", message=recipe, new_connection=True - # ) - else: - mock_logger.error.assert_called_with("No TransportManager object was set up") + mock_transport_object.send.assert_called_with( + queue="processing_recipe", message=recipe, new_connection=True + )