From cb1a877044d7105974b201bd3ae71f297a6613d2 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Tue, 18 Apr 2023 10:14:41 +0100 Subject: [PATCH 01/22] Changed Point3D and Point2D to use numpy arrays --- .../experiment_plans/fast_grid_scan_plan.py | 19 ++++++++++--------- .../unit_tests/test_fast_grid_scan_plan.py | 4 ++-- src/artemis/utils.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/artemis/experiment_plans/fast_grid_scan_plan.py b/src/artemis/experiment_plans/fast_grid_scan_plan.py index 8009a93c3..c11fb5183 100644 --- a/src/artemis/experiment_plans/fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/fast_grid_scan_plan.py @@ -5,6 +5,7 @@ import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp +import numpy as np from bluesky import RunEngine from bluesky.utils import ProgressBarManager from dodal import i03 @@ -39,7 +40,7 @@ SIM_BEAMLINE, ) from artemis.tracing import TRACER -from artemis.utils import Point3D +from artemis.utils import create_point if TYPE_CHECKING: from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( @@ -143,7 +144,7 @@ def read_hardware_for_ispyb( @bpp.run_decorator(md={"subplan_name": "move_xyz"}) def move_xyz( sample_motors, - xray_centre_motor_position: Point3D, + xray_centre_motor_position: np.ndarray, md={ "plan_name": "move_xyz", }, @@ -152,12 +153,12 @@ def move_xyz( from gridscan processing results)""" artemis.log.LOGGER.info(f"Moving Smargon x, y, z to: {xray_centre_motor_position}") yield from bps.mv( - sample_motors.x, - xray_centre_motor_position.x, - sample_motors.y, - xray_centre_motor_position.y, - sample_motors.z, - xray_centre_motor_position.z, + sample_motors[0], + xray_centre_motor_position[0], + sample_motors[1], + xray_centre_motor_position[1], + sample_motors[2], + xray_centre_motor_position[2], ) @@ -239,7 +240,7 @@ def run_gridscan_and_move( and moves to the centre of mass determined by zocalo""" # We get the initial motor positions so we can return to them on zocalo failure - initial_xyz = Point3D( + initial_xyz = create_point( (yield from bps.rd(fgs_composite.sample_motors.x)), (yield from bps.rd(fgs_composite.sample_motors.y)), (yield from bps.rd(fgs_composite.sample_motors.z)), diff --git a/src/artemis/unit_tests/test_fast_grid_scan_plan.py b/src/artemis/unit_tests/test_fast_grid_scan_plan.py index 97fda58f8..1e97b33ec 100644 --- a/src/artemis/unit_tests/test_fast_grid_scan_plan.py +++ b/src/artemis/unit_tests/test_fast_grid_scan_plan.py @@ -39,7 +39,7 @@ from artemis.log import set_up_logging_handlers from artemis.parameters.external_parameters import RawParameters from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import Point3D, create_point @pytest.fixture @@ -236,7 +236,7 @@ def test_results_passed_to_move_motors( set_up_logging_handlers(logging_level="INFO", dev_mode=True) RE.subscribe(VerbosePlanExecutionLoggingCallback()) motor_position = test_params.experiment_params.grid_position_to_motor_position( - Point3D(1, 2, 3) + create_point(1, 2, 3) ) RE(move_xyz(fake_fgs_composite.sample_motors, motor_position)) bps_mv.assert_called_once_with( diff --git a/src/artemis/utils.py b/src/artemis/utils.py index fe8ffda5c..f89a9c8ea 100644 --- a/src/artemis/utils.py +++ b/src/artemis/utils.py @@ -1,4 +1,16 @@ from collections import namedtuple +import numpy as np + + +def create_point(*args): + if len(args) == 2: + return np.array([args[0], args[1]], dtype=np.int8) + elif len(args) == 3: + return np.array([args[0], args[1], args[2]], dtype=np.int8) + else: + raise AttributeError("test") + + Point2D = namedtuple("Point2D", ["x", "y"]) Point3D = namedtuple("Point3D", ["x", "y", "z"]) From fb3c0cd86ab5e75dd133842ef9d747343420d9e1 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 19 Apr 2023 11:34:26 +0100 Subject: [PATCH 02/22] Fixed tests, create_point checks for NoneType --- .../experiment_plans/fast_grid_scan_plan.py | 10 +++++----- .../unit_tests/test_fast_grid_scan_plan.py | 17 +++++++++++++---- src/artemis/utils.py | 7 ++++++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/artemis/experiment_plans/fast_grid_scan_plan.py b/src/artemis/experiment_plans/fast_grid_scan_plan.py index c11fb5183..3a28135fd 100644 --- a/src/artemis/experiment_plans/fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/fast_grid_scan_plan.py @@ -5,7 +5,6 @@ import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp -import numpy as np from bluesky import RunEngine from bluesky.utils import ProgressBarManager from dodal import i03 @@ -23,6 +22,7 @@ Undulator, Zebra, ) +from numpy import ndarray import artemis.log from artemis.device_setup_plans.setup_zebra_for_fgs import ( @@ -144,7 +144,7 @@ def read_hardware_for_ispyb( @bpp.run_decorator(md={"subplan_name": "move_xyz"}) def move_xyz( sample_motors, - xray_centre_motor_position: np.ndarray, + xray_centre_motor_position: ndarray, md={ "plan_name": "move_xyz", }, @@ -153,11 +153,11 @@ def move_xyz( from gridscan processing results)""" artemis.log.LOGGER.info(f"Moving Smargon x, y, z to: {xray_centre_motor_position}") yield from bps.mv( - sample_motors[0], + sample_motors.x, xray_centre_motor_position[0], - sample_motors[1], + sample_motors.y, xray_centre_motor_position[1], - sample_motors[2], + sample_motors.z, xray_centre_motor_position[2], ) diff --git a/src/artemis/unit_tests/test_fast_grid_scan_plan.py b/src/artemis/unit_tests/test_fast_grid_scan_plan.py index 1e97b33ec..b2b5800a0 100644 --- a/src/artemis/unit_tests/test_fast_grid_scan_plan.py +++ b/src/artemis/unit_tests/test_fast_grid_scan_plan.py @@ -2,6 +2,7 @@ from unittest.mock import ANY, MagicMock, call, patch import bluesky.plan_stubs as bps +import numpy as np import pytest from bluesky.run_engine import RunEngine from dodal.devices.aperturescatterguard import AperturePositions @@ -39,7 +40,7 @@ from artemis.log import set_up_logging_handlers from artemis.parameters.external_parameters import RawParameters from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D, create_point +from artemis.utils import create_point @pytest.fixture @@ -240,7 +241,7 @@ def test_results_passed_to_move_motors( ) RE(move_xyz(fake_fgs_composite.sample_motors, motor_position)) bps_mv.assert_called_once_with( - ANY, motor_position.x, ANY, motor_position.y, ANY, motor_position.z + ANY, motor_position[0], ANY, motor_position[1], ANY, motor_position[2] ) @@ -275,7 +276,11 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( ) run_gridscan.assert_called_once_with(fake_fgs_composite, params) - move_xyz.assert_called_once_with(ANY, Point3D(0.05, 0.15000000000000002, 0.25)) + array_arg = move_xyz.call_args.args[1] + np.testing.assert_array_almost_equal( + array_arg, create_point(0.05, 0.15000000000000002, 0.25) + ) + move_xyz.assert_called_once() @patch( @@ -308,7 +313,11 @@ def test_logging_within_plan( ) run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) - move_xyz.assert_called_once_with(ANY, Point3D(0.05, 0.15000000000000002, 0.25)) + array_arg = move_xyz.call_args.args[1] + np.testing.assert_array_almost_equal( + array_arg, create_point(0.05, 0.15000000000000002, 0.25) + ) + move_xyz.assert_called_once() @patch("artemis.experiment_plans.fast_grid_scan_plan.bps.sleep") diff --git a/src/artemis/utils.py b/src/artemis/utils.py index f89a9c8ea..40d22efee 100644 --- a/src/artemis/utils.py +++ b/src/artemis/utils.py @@ -4,12 +4,17 @@ def create_point(*args): + args = list(args) + for index, arg in enumerate(args): + if args[index] is None: + args[index] = 0 + if len(args) == 2: return np.array([args[0], args[1]], dtype=np.int8) elif len(args) == 3: return np.array([args[0], args[1], args[2]], dtype=np.int8) else: - raise AttributeError("test") + raise TypeError("Invalid number of arguments") Point2D = namedtuple("Point2D", ["x", "y"]) From 2cfa8fb88bdf11dff5b851ca271d3dcc294fb65c Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 19 Apr 2023 11:54:48 +0100 Subject: [PATCH 03/22] changed create_point dtype --- src/artemis/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/artemis/utils.py b/src/artemis/utils.py index 40d22efee..bf6ed3958 100644 --- a/src/artemis/utils.py +++ b/src/artemis/utils.py @@ -10,9 +10,9 @@ def create_point(*args): args[index] = 0 if len(args) == 2: - return np.array([args[0], args[1]], dtype=np.int8) + return np.array([args[0], args[1]], dtype=np.float16) elif len(args) == 3: - return np.array([args[0], args[1], args[2]], dtype=np.int8) + return np.array([args[0], args[1], args[2]], dtype=np.float16) else: raise TypeError("Invalid number of arguments") From 3d01051f5f003540e694d5df22716559eb002441 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 19 Apr 2023 13:21:40 +0100 Subject: [PATCH 04/22] Replace Point3D with create_point --- .../fgs/tests/test_fgs_callback_collection.py | 4 ++-- .../fgs/tests/test_zocalo_handler.py | 24 +++++++++---------- .../callbacks/fgs/zocalo_callback.py | 17 ++++++------- .../ispyb/ispyb_dataclass.py | 15 ++++++------ .../system_tests/conftest.py | 4 ++-- .../system_tests/test_zocalo_system.py | 6 ++--- 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py index 040fd0da3..716af9ac1 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py @@ -13,7 +13,7 @@ ) from artemis.parameters.constants import SIM_BEAMLINE from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point def test_callback_collection_init(): @@ -89,7 +89,7 @@ def test_communicator_in_composite_run( callbacks.zocalo_handler._wait_for_result = MagicMock() callbacks.zocalo_handler._run_end = MagicMock() callbacks.zocalo_handler._run_start = MagicMock() - callbacks.zocalo_handler.xray_centre_motor_position = Point3D(1, 2, 3) + callbacks.zocalo_handler.xray_centre_motor_position = create_point(1, 2, 3) fast_grid_scan_composite = FGSComposite() # this is where it's currently getting stuck: diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index 51e47b990..70f5a1375 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -10,7 +10,7 @@ from artemis.external_interaction.exceptions import ISPyBDepositionNotMade from artemis.external_interaction.zocalo.zocalo_interaction import NoDiffractionFound from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -75,7 +75,7 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal callbacks = FGSCallbackCollection.from_params(params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) - expected_centre_grid_coords = Point3D(1, 2, 3) + expected_centre_grid_coords = create_point(1, 2, 3) single_crystal_result = [ { "max_voxel": [1, 2, 3], @@ -88,16 +88,16 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal single_crystal_result ) - found_centre = callbacks.zocalo_handler.wait_for_results(Point3D(0, 0, 0))[0] + found_centre = callbacks.zocalo_handler.wait_for_results(create_point(0, 0, 0))[0] callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( 100 ) expected_centre_motor_coords = ( params.experiment_params.grid_position_to_motor_position( - Point3D( - expected_centre_grid_coords.x - 0.5, - expected_centre_grid_coords.y - 0.5, - expected_centre_grid_coords.z - 0.5, + create_point( + expected_centre_grid_coords[0] - 0.5, + expected_centre_grid_coords[1] - 0.5, + expected_centre_grid_coords[2] - 0.5, ) ) ) @@ -113,7 +113,7 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ NoDiffractionFound() ) - fallback_position = Point3D(1, 2, 3) + fallback_position = create_point(1, 2, 3) found_centre = callbacks.zocalo_handler.wait_for_results(fallback_position)[0] callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( @@ -136,11 +136,11 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b callbacks = FGSCallbackCollection.from_params(params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) - expected_centre_grid_coords = Point3D(4, 6, 2) + expected_centre_grid_coords = create_point(4, 6, 2) multi_crystal_result = [ { "max_voxel": [1, 2, 3], - "centre_of_mass": Point3D(3, 11, 11), + "centre_of_mass": create_point(3, 11, 11), "bounding_box": [[1, 1, 1], [3, 3, 3]], "n_voxels": 2, "total_count": 192512.0, @@ -157,14 +157,14 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b multi_crystal_result ) found_centre, found_bbox = callbacks.zocalo_handler.wait_for_results( - Point3D(0, 0, 0) + create_point(0, 0, 0) ) callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( 100 ) expected_centre_motor_coords = ( params.experiment_params.grid_position_to_motor_position( - Point3D( + create_point( expected_centre_grid_coords.x - 0.5, expected_centre_grid_coords.y - 0.5, expected_centre_grid_coords.z - 0.5, diff --git a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py index 2e047216e..32d55e247 100644 --- a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py @@ -5,6 +5,7 @@ from typing import Callable, Optional from bluesky.callbacks import CallbackBase +from numpy import ndarray from artemis.external_interaction.callbacks.fgs.ispyb_callback import ( FGSISPyBHandlerCallback, @@ -16,7 +17,7 @@ ) from artemis.log import LOGGER from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point class FGSZocaloCallback(CallbackBase): @@ -43,7 +44,7 @@ def __init__( self, parameters: "InternalParameters", ispyb_handler: FGSISPyBHandlerCallback ): self.grid_position_to_motor_position: Callable[ - [Point3D], Point3D + [ndarray], ndarray ] = parameters.experiment_params.grid_position_to_motor_position self.processing_start_time = 0.0 self.processing_time = 0.0 @@ -76,14 +77,14 @@ def stop(self, doc: dict): self.zocalo_interactor.run_end(id) self.processing_start_time = time.time() - def wait_for_results(self, fallback_xyz: Point3D) -> Point3D: + def wait_for_results(self, fallback_xyz: ndarray) -> tuple[ndarray, Optional[list]]: """Blocks until a centre has been received from Zocalo Args: - fallback_xyz (Point3D): The position to fallback to if no centre is found + fallback_xyz (ndarray): The position to fallback to if no centre is found Returns: - Point3D: The xray centre position to move to + ndarray: The xray centre position to move to """ datacollection_group_id = self.ispyb.ispyb_ids[2] @@ -117,11 +118,11 @@ def wait_for_results(self, fallback_xyz: Point3D) -> Point3D: ) self.ispyb.append_to_comment(crystal_summary) - raw_centre = Point3D(*(raw_results[0]["centre_of_mass"])) + raw_centre = create_point(*(raw_results[0]["centre_of_mass"])) # _wait_for_result returns the centre of the grid box, but we want the corner - results = Point3D( - raw_centre.x - 0.5, raw_centre.y - 0.5, raw_centre.z - 0.5 + results = create_point( + raw_centre[0] - 0.5, raw_centre[1] - 0.5, raw_centre[2] - 0.5 ) xray_centre = self.grid_position_to_motor_position(results) diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 61848f2db..79dc369e1 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -3,8 +3,9 @@ from typing import List, Optional from dataclasses_json import config, dataclass_json +from numpy import ndarray -from artemis.utils import Point3D +from artemis.utils import create_point ISPYB_PARAM_DEFAULTS = { "sample_id": None, @@ -13,8 +14,8 @@ "microns_per_pixel_x": 0.0, "microns_per_pixel_y": 0.0, # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels - "upper_left": Point3D(x=0, y=0, z=0), - "position": Point3D(x=0, y=0, z=0), + "upper_left": create_point(x=0, y=0, z=0), + "position": create_point(x=0, y=0, z=0), "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], "transmission": 1.0, @@ -40,19 +41,19 @@ class IspybParams: microns_per_pixel_x: float microns_per_pixel_y: float - upper_left: Point3D = field( + upper_left: ndarray = field( # in px on the image metadata=config( encoder=lambda mytuple: mytuple._asdict(), - decoder=lambda mydict: Point3D(**mydict), + decoder=lambda mydict: create_point(**mydict), ) ) - position: Point3D = field( + position: ndarray = field( # motor position metadata=config( encoder=lambda mytuple: mytuple._asdict(), - decoder=lambda mydict: Point3D(**mydict), + decoder=lambda mydict: create_point(**mydict), ) ) diff --git a/src/artemis/external_interaction/system_tests/conftest.py b/src/artemis/external_interaction/system_tests/conftest.py index 23eda3033..3feacf99b 100644 --- a/src/artemis/external_interaction/system_tests/conftest.py +++ b/src/artemis/external_interaction/system_tests/conftest.py @@ -14,7 +14,7 @@ StoreInIspyb3D, ) from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point ISPYB_CONFIG = "/dls_sw/dasc/mariadb/credentials/ispyb-dev.cfg" @@ -76,7 +76,7 @@ def fetch_comment() -> Callable: @pytest.fixture def dummy_params(): dummy_params = InternalParameters() - dummy_params.artemis_params.ispyb_params.upper_left = Point3D(100, 100, 50) + dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 dummy_params.artemis_params.ispyb_params.visit_path = ( diff --git a/src/artemis/external_interaction/system_tests/test_zocalo_system.py b/src/artemis/external_interaction/system_tests/test_zocalo_system.py index b1bb3cc9b..e5d45ff98 100644 --- a/src/artemis/external_interaction/system_tests/test_zocalo_system.py +++ b/src/artemis/external_interaction/system_tests/test_zocalo_system.py @@ -9,7 +9,7 @@ TEST_RESULT_SMALL, ) from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point @pytest.mark.s03 @@ -28,7 +28,7 @@ def test_when_running_start_stop_then_get_expected_returned_results(zocalo_env): @pytest.fixture def run_zocalo_with_dev_ispyb(dummy_params: InternalParameters, dummy_ispyb_3d): - def inner(sample_name="", fallback=Point3D(0, 0, 0)): + def inner(sample_name="", fallback=create_point(0, 0, 0)): dummy_params.artemis_params.detector_params.prefix = sample_name zc: FGSZocaloCallback = FGSCallbackCollection.from_params( dummy_params @@ -48,7 +48,7 @@ def inner(sample_name="", fallback=Point3D(0, 0, 0)): def test_given_a_result_with_no_diffraction_when_zocalo_called_then_move_to_fallback( run_zocalo_with_dev_ispyb, zocalo_env ): - fallback = Point3D(1, 2, 3) + fallback = create_point(1, 2, 3) zc, centre = run_zocalo_with_dev_ispyb("NO_DIFF", fallback) assert centre == fallback From c6884af2f679629e90dc4f136c6e401239c09b9d Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 19 Apr 2023 14:02:35 +0100 Subject: [PATCH 05/22] Fix tests and replace Point2D --- src/artemis/external_interaction/ispyb/store_in_ispyb.py | 6 +++--- .../unit_tests/test_store_in_ispyb.py | 8 ++++---- .../unit_tests/test_zocalo_interaction.py | 9 ++++++--- .../external_interaction/zocalo/zocalo_interaction.py | 5 +++-- src/artemis/parameters/tests/test_internal_parameters.py | 6 +++--- src/artemis/utils.py | 6 ------ 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/artemis/external_interaction/ispyb/store_in_ispyb.py b/src/artemis/external_interaction/ispyb/store_in_ispyb.py index 6110b8f1a..c501f3d64 100755 --- a/src/artemis/external_interaction/ispyb/store_in_ispyb.py +++ b/src/artemis/external_interaction/ispyb/store_in_ispyb.py @@ -13,7 +13,7 @@ from artemis.log import LOGGER from artemis.parameters.internal_parameters import InternalParameters from artemis.tracing import TRACER -from artemis.utils import Point2D +from artemis.utils import create_point I03_EIGER_DETECTOR = 78 EIGER_FILE_SUFFIX = "h5" @@ -81,7 +81,7 @@ def store_grid_scan(self, full_params: InternalParameters): self.run_number = self.detector_params.run_number self.omega_start = self.detector_params.omega_start self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_start - self.upper_left = Point2D( + self.upper_left = create_point( self.ispyb_params.upper_left.x, self.ispyb_params.upper_left.y, ) @@ -311,7 +311,7 @@ def __prepare_second_scan_params(self): self.omega_start += 90 self.run_number += 1 self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_end - self.upper_left = Point2D( + self.upper_left = create_point( self.ispyb_params.upper_left.x, self.ispyb_params.upper_left.z, ) diff --git a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py index fdd8eff70..a1b3413d7 100644 --- a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py +++ b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py @@ -11,7 +11,7 @@ ) from artemis.parameters.constants import SIM_ISPYB_CONFIG from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point TEST_DATA_COLLECTION_IDS = [12, 13] TEST_DATA_COLLECTION_GROUP_ID = 34 @@ -25,7 +25,7 @@ @pytest.fixture def dummy_params(): dummy_params = InternalParameters() - dummy_params.artemis_params.ispyb_params.upper_left = Point3D(100, 100, 50) + dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 return dummy_params @@ -126,7 +126,7 @@ def test_store_3d_grid_scan( y = 1 z = 2 - dummy_params.artemis_params.ispyb_params.upper_left = Point3D(x, y, z) + dummy_params.artemis_params.ispyb_params.upper_left = create_point(x, y, z) dummy_params.experiment_params.z_step_size = 0.2 assert dummy_ispyb_3d.experiment_type == "Mesh3D" @@ -312,7 +312,7 @@ def test_ispyb_deposition_rounds_to_int( mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition ) mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection - dummy_ispyb.full_params.artemis_params.ispyb_params.upper_left = Point3D( + dummy_ispyb.full_params.artemis_params.ispyb_params.upper_left = create_point( 0.01, 100, 50 ) dummy_ispyb.begin_deposition() diff --git a/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py b/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py index 0893d6546..ee76be307 100644 --- a/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py +++ b/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py @@ -6,6 +6,7 @@ from typing import Callable, Dict from unittest.mock import MagicMock, patch +import numpy as np import pytest from pytest import mark, raises from zocalo.configuration import Configuration @@ -15,7 +16,7 @@ ZocaloInteractor, ) from artemis.parameters.constants import SIM_ZOCALO_ENV -from artemis.utils import Point3D +from artemis.utils import create_point EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -137,8 +138,10 @@ def test_when_message_recieved_from_zocalo_then_point_returned( return_value = future.result() assert type(return_value) == list - returned_com = Point3D(*return_value[0]["centre_of_mass"]) - assert returned_com == Point3D(*centre_of_mass_coords) + returned_com = create_point(*return_value[0]["centre_of_mass"]) + np.testing.assert_array_almost_equal( + returned_com, create_point(*centre_of_mass_coords) + ) @patch("workflows.recipe.wrap_subscribe") diff --git a/src/artemis/external_interaction/zocalo/zocalo_interaction.py b/src/artemis/external_interaction/zocalo/zocalo_interaction.py index b4968aaf9..c36c10cf2 100644 --- a/src/artemis/external_interaction/zocalo/zocalo_interaction.py +++ b/src/artemis/external_interaction/zocalo/zocalo_interaction.py @@ -5,6 +5,8 @@ from time import sleep from typing import Optional +from numpy import ndarray + import workflows.recipe import workflows.transport import zocalo.configuration @@ -12,7 +14,6 @@ import artemis.log from artemis.exceptions import WarningException -from artemis.utils import Point3D TIMEOUT = 90 @@ -82,7 +83,7 @@ def run_end(self, data_collection_id: int): def wait_for_result( self, data_collection_group_id: int, timeout: int = None - ) -> Point3D: + ) -> ndarray: """Block until a result is received from Zocalo. Args: data_collection_group_id (int): The ID of the data collection group representing diff --git a/src/artemis/parameters/tests/test_internal_parameters.py b/src/artemis/parameters/tests/test_internal_parameters.py index a383fbdee..5a7ef3e1d 100644 --- a/src/artemis/parameters/tests/test_internal_parameters.py +++ b/src/artemis/parameters/tests/test_internal_parameters.py @@ -3,7 +3,7 @@ from artemis.parameters.external_parameters import RawParameters from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils import Point3D +from artemis.utils import create_point def test_parameters_load_from_file(): @@ -16,8 +16,8 @@ def test_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - assert ispyb_params.position == Point3D(10, 20, 30) - assert ispyb_params.upper_left == Point3D(10, 20, 30) + assert ispyb_params.position == create_point(10, 20, 30) + assert ispyb_params.upper_left == create_point(10, 20, 30) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/utils.py b/src/artemis/utils.py index bf6ed3958..9deacfc01 100644 --- a/src/artemis/utils.py +++ b/src/artemis/utils.py @@ -1,5 +1,3 @@ -from collections import namedtuple - import numpy as np @@ -15,7 +13,3 @@ def create_point(*args): return np.array([args[0], args[1], args[2]], dtype=np.float16) else: raise TypeError("Invalid number of arguments") - - -Point2D = namedtuple("Point2D", ["x", "y"]) -Point3D = namedtuple("Point3D", ["x", "y", "z"]) From cbfba17185b716484051247916077778c84c4062 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 19 Apr 2023 16:02:22 +0100 Subject: [PATCH 06/22] minor fixes --- .../ispyb/ispyb_dataclass.py | 4 ++-- .../bad_test_parameters_wrong_version.json | 20 +++++++++---------- .../tests/test_data/good_test_parameters.json | 20 +++++++++---------- .../good_test_rotation_scan_parameters.json | 20 +++++++++---------- .../tests/test_internal_parameters.py | 1 + test_parameters.json | 20 +++++++++---------- 6 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 79dc369e1..2116e86c5 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -14,8 +14,8 @@ "microns_per_pixel_x": 0.0, "microns_per_pixel_y": 0.0, # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels - "upper_left": create_point(x=0, y=0, z=0), - "position": create_point(x=0, y=0, z=0), + "upper_left": create_point(0, 0, 0), + "position": create_point(0, 0, 0), "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], "transmission": 1.0, diff --git a/src/artemis/parameters/tests/test_data/bad_test_parameters_wrong_version.json b/src/artemis/parameters/tests/test_data/bad_test_parameters_wrong_version.json index 3efcad72b..c5afa3bec 100644 --- a/src/artemis/parameters/tests/test_data/bad_test_parameters_wrong_version.json +++ b/src/artemis/parameters/tests/test_data/bad_test_parameters_wrong_version.json @@ -21,16 +21,16 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, - "position": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, + "upper_left": [ + 10.0, + 20.0, + 30.0 + ], + "position": [ + 10.0, + 20.0, + 30.0 + ], "xtal_snapshots_omega_start": [ "test_1_y", "test_2_y", diff --git a/src/artemis/parameters/tests/test_data/good_test_parameters.json b/src/artemis/parameters/tests/test_data/good_test_parameters.json index 4c8827b60..299fa8bb8 100644 --- a/src/artemis/parameters/tests/test_data/good_test_parameters.json +++ b/src/artemis/parameters/tests/test_data/good_test_parameters.json @@ -17,16 +17,16 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, - "position": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, + "upper_left": [ + 10.0, + 20.0, + 30.0 + ], + "position": [ + 10.0, + 20.0, + 30.0 + ], "xtal_snapshots_omega_start": [ "test_1_y", "test_2_y", diff --git a/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json b/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json index 52e0741e8..44e2196a3 100644 --- a/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json +++ b/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json @@ -17,16 +17,16 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, - "position": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, + "upper_left": [ + 10.0, + 20.0, + 30.0 + ], + "position": [ + 10.0, + 20.0, + 30.0 + ], "xtal_snapshots_omega_start": [ "test_1_y", "test_2_y", diff --git a/src/artemis/parameters/tests/test_internal_parameters.py b/src/artemis/parameters/tests/test_internal_parameters.py index 5a7ef3e1d..a3457fe26 100644 --- a/src/artemis/parameters/tests/test_internal_parameters.py +++ b/src/artemis/parameters/tests/test_internal_parameters.py @@ -3,6 +3,7 @@ from artemis.parameters.external_parameters import RawParameters from artemis.parameters.internal_parameters import InternalParameters + from artemis.utils import create_point diff --git a/test_parameters.json b/test_parameters.json index 4c8827b60..299fa8bb8 100644 --- a/test_parameters.json +++ b/test_parameters.json @@ -17,16 +17,16 @@ "visit_path": "/tmp/cm31105-4/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, - "upper_left": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, - "position": { - "x": 10.0, - "y": 20.0, - "z": 30.0 - }, + "upper_left": [ + 10.0, + 20.0, + 30.0 + ], + "position": [ + 10.0, + 20.0, + 30.0 + ], "xtal_snapshots_omega_start": [ "test_1_y", "test_2_y", From 018a2f8be0491ac0886ca4bd4086d7762c49fce9 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Thu, 20 Apr 2023 10:17:13 +0100 Subject: [PATCH 07/22] Merge main into 348_replace_point_3d_to_support_arithmetic --- .coveragerc | 18 +- .vscode/launch.json | 15 + README.md | 2 +- run_artemis.sh | 117 +++---- setup.cfg | 8 - src/artemis/__main__.py | 62 +++- .../experiment_plans/experiment_registry.py | 16 +- .../experiment_plans/fast_grid_scan_plan.py | 36 ++- .../tests/test_experiment_registry.py | 7 +- .../callbacks/aperture_change_callback.py | 15 + .../callbacks/fgs/ispyb_callback.py | 6 +- .../callbacks/fgs/nexus_callback.py | 6 +- .../fgs/tests/test_fgs_callback_collection.py | 11 +- .../callbacks/fgs/tests/test_ispyb_handler.py | 27 +- .../callbacks/fgs/tests/test_nexus_handler.py | 28 +- .../fgs/tests/test_zocalo_handler.py | 48 ++- .../callbacks/fgs/zocalo_callback.py | 6 +- .../ispyb/store_in_ispyb.py | 4 +- .../system_tests/conftest.py | 7 +- .../system_tests/test_ispyb_dev_connection.py | 8 +- .../system_tests/test_zocalo_system.py | 10 +- .../unit_tests/test_store_in_ispyb.py | 13 +- .../unit_tests/test_write_nexus.py | 19 +- src/artemis/parameters/beamline_parameters.py | 10 +- src/artemis/parameters/constants.py | 2 +- src/artemis/parameters/external_parameters.py | 205 ++---------- src/artemis/parameters/internal_parameters.py | 142 --------- .../internal_parameters/__init__.py | 5 + .../internal_parameters.py | 213 +++++++++++++ .../plan_specific/fgs_internal_params.py | 18 ++ .../rotation_scan_internal_params.py | 69 ++++ .../tests/test_fgs_internal_parameters.py | 26 ++ .../test_rotation_internal_parameters.py | 72 +++++ .../rotation_scan_params_schema.json | 9 + .../full_external_parameters_schema.json | 2 +- .../schemas/ispyb_parameters_schema.json | 78 +---- .../tests/test_data/good_test_parameters.json | 2 +- .../good_test_rotation_scan_parameters.json | 7 +- .../test_data/test_beamline_parameters.txt | 299 ++++++++++++++++++ .../tests/test_external_parameters.py | 110 ++----- .../tests/test_internal_parameters.py | 188 ++++++++++- .../test_aperturescatterguard_system.py | 135 +------- .../test_device_setups_and_cleanups.py | 34 +- src/artemis/system_tests/test_fgs_plan.py | 42 ++- src/artemis/system_tests/test_main_system.py | 82 ++++- .../unit_tests/test_fast_grid_scan_plan.py | 39 +-- test_parameter_defaults.json | 74 +++++ test_parameters.json | 2 +- 48 files changed, 1495 insertions(+), 859 deletions(-) mode change 100644 => 100755 src/artemis/__main__.py mode change 100644 => 100755 src/artemis/experiment_plans/fast_grid_scan_plan.py create mode 100644 src/artemis/external_interaction/callbacks/aperture_change_callback.py delete mode 100644 src/artemis/parameters/internal_parameters.py create mode 100644 src/artemis/parameters/internal_parameters/__init__.py create mode 100644 src/artemis/parameters/internal_parameters/internal_parameters.py create mode 100644 src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py create mode 100644 src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py create mode 100644 src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py create mode 100644 src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py create mode 100644 src/artemis/parameters/tests/test_data/test_beamline_parameters.txt create mode 100644 test_parameter_defaults.json diff --git a/.coveragerc b/.coveragerc index 66961672c..efb144952 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,18 @@ [run] -omit = */test_* +omit = + */test_* + **/conftest.py +data_file = /tmp/python-artemis.coverage + +[report] +exclude_also = + if TYPE_CHECKING: + def __repr__ + raise NotImplementedError + @(abc\.)?abstractmethod + +[paths] +# Tests are run from installed location, map back to the src directory +source = + src + **/site-packages/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index f33dd3c45..7bdbe0f09 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,21 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "Python: Run Artemis in dev mode", + "type": "python", + "request": "launch", + "module": "artemis", + "args": [ + "--dev", + "--verbose-event-logging", + "--skip_startup_connection" + ], + "env": { + "EPICS_CA_SERVER_PORT": "5066" + }, + "justMyCode": false + }, { "name": "Python: Current File", "type": "python", diff --git a/README.md b/README.md index fb5472ed7..2b174473f 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ python -m artemis --dev --verbose-event-logging Lastly, you can choose to skip running the hardware connection scripts on startup with the flag ``` -python -m artemis --skip_startup_connection +python -m artemis --skip-startup-connection ``` Testing diff --git a/run_artemis.sh b/run_artemis.sh index 1618fb6bc..720b687e7 100755 --- a/run_artemis.sh +++ b/run_artemis.sh @@ -1,40 +1,10 @@ #!/bin/bash -# Params - 2 semver strings of the form 1.11.2.3 -# Returns - 0 if equal version, 1 if 1st param is a greater version number, 2 if 2nd param has greater version number -checkver () { - if [[ $1 == $2 ]] - then - return 0 - fi - local IFS=. - local i ver1=($1) ver2=($2) - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) - do - ver1[i]=0 - done - for ((i=0; i<${#ver1[@]}; i++)) - do - if [[ -z ${ver2[i]} ]] - then - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})) - then - return 1 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})) - then - return 2 - fi - done - return 0 -} - STOP=0 START=1 -DEPLOY=0 -SKIP_STARTUP_CONNECTION=0 +SKIP_STARTUP_CONNECTION=false +VERBOSE_EVENT_LOGGING=false +LOGGING_LEVEL="INFO" for option in "$@"; do case $option in @@ -42,33 +12,36 @@ for option in "$@"; do BEAMLINE="${option#*=}" shift ;; - -v=*|--version=*) - VERSION="${option#*=}" - shift - ;; --stop) STOP=1 ;; --no-start) START=0 ;; - --deploy) - DEPLOY=1 + --skip-startup-connection) + SKIP_STARTUP_CONNECTION=true ;; - --skip_startup_connection) - SKIP_STARTUP_CONNECTION=1 + --dev) + IN_DEV=true ;; - --help|--info) - echo "Options" + --verbose-event-logging) + VERBOSE_EVENT_LOGGING=true + ;; + --logging-level=*) + LOGGING_LEVEL="${option#*=}" + ;; + + --help|--info|--h) + + #Combine help from here and help from artemis + source .venv/bin/activate + python -m artemis --help echo " -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline" - echo " -v, --version=VERSION Specifies the artemis version number to deploy. Option should be given in the form 0.0.0.0" - echo " Will check git tags and use the lastest version as a default if no version is specified." - echo " Unused outside of deploy operation." echo " " echo "Operations" echo " --stop Used to stop a currently running instance of Artemis. Will override any other operations" echo " options" - echo " --deploy Used to update and install a new version of Artemis." + echo " --no-start Used to specify that the script should be run without starting the server." echo " " echo "By default this script will start an Artemis server unless the --no-start flag is specified." @@ -81,6 +54,13 @@ for option in "$@"; do esac done +#Check valid logging level was chosen +if [[ "$LOGGING_LEVEL" != "INFO" && "$LOGGING_LEVEL" != "CRITICAL" && "$LOGGING_LEVEL" != "ERROR" + && "$LOGGING_LEVEL" != "WARNING" && "$LOGGING_LEVEL" != "DEBUG" ]]; then + echo "Invalid logging level selected, defaulting to INFO" + LOGGING_LEVEL="INFO" +fi + if [ -z "${BEAMLINE}" ]; then echo "BEAMLINE parameter not set, assuming running on a dev machine." echo "If you would like to run not in dev use the option -b, --beamline=BEAMLNE to set it manually" @@ -103,35 +83,6 @@ if [[ $STOP == 1 ]]; then exit 0 fi -if [[ $DEPLOY == 1 ]]; then - git fetch --all --tags --prune - if [[ -z "${VERSION}" ]]; then - VERSION="0" - for version_tag in $(git ls-remote --tags origin/main); do - checkver $VERSION ${version_tag} - case $? in - 0|1) ;; # do nothing if VERSION is still the latest version - 2) VERSION = ${version_tag} ;; - esac - done - fi - - git checkout "tags/${VERSION}" - - module unload controls_dev - module load python/3.10 - - if [ -d "./.venv" ] - then - rm -rf .venv - fi - mkdir .venv - - python -m venv .venv - - pip install -e . -fi - if [[ $START == 1 ]]; then if [ $IN_DEV == false ]; then if [[ $HOSTNAME != "${BEAMLINE}-control.diamond.ac.uk" || $USER != "gda2" ]]; then @@ -168,7 +119,19 @@ if [[ $START == 1 ]]; then source .venv/bin/activate - python -m artemis `if [ $IN_DEV == true ]; then echo "--dev"; fi` `if [ $SKIP_STARTUP_CONNECTION == 1 ]; then echo "--skip_startup_connection"; fi`>$start_log_path 2>&1 & + #Add future arguments here + declare -A args=( ["IN_DEV"]="$IN_DEV" ["SKIP_STARTUP_CONNECTION"]="$SKIP_STARTUP_CONNECTION" ["VERBOSE_EVENT_LOGGING"]="$VERBOSE_EVENT_LOGGING" + ["LOGGING_LEVEL"]="$LOGGING_LEVEL") + declare -A arg_strings=( ["IN_DEV"]="--dev" ["SKIP_STARTUP_CONNECTION"]="--skip-startup-connection" ["VERBOSE_EVENT_LOGGING"]="--verbose-event-logging" + ["LOGGING_LEVEL"]="--logging-level=$LOGGING_LEVEL") + + commands=() + for i in "${!args[@]}" + do + if [ "${args[$i]}" != false ]; then commands+="${arg_strings[$i]} "; fi; + done + + python -m artemis `echo $commands;`>$start_log_path 2>&1 & echo "Waiting for Artemis to boot" diff --git a/setup.cfg b/setup.cfg index a5f5831ac..7a0cd45f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,11 +83,3 @@ extend-ignore = # Ignore calls to dict()/tuple() instead of using {}/() C408, -[coverage:run] -data_file = /tmp/python-artemis.coverage - -[coverage:paths] -# Tests are run from installed location, map back to the src directory -source = - src - **/site-packages/ diff --git a/src/artemis/__main__.py b/src/artemis/__main__.py old mode 100644 new mode 100755 index 4b5e6affd..6f3798dc0 --- a/src/artemis/__main__.py +++ b/src/artemis/__main__.py @@ -4,16 +4,21 @@ from dataclasses import dataclass from json import JSONDecodeError from queue import Queue +from traceback import format_exception from typing import Callable, Optional, Tuple from bluesky import RunEngine from dataclasses_json import dataclass_json from flask import Flask, request from flask_restful import Api, Resource +from jsonschema.exceptions import ValidationError import artemis.log from artemis.exceptions import WarningException from artemis.experiment_plans.experiment_registry import PLAN_REGISTRY, PlanNotFound +from artemis.external_interaction.callbacks.aperture_change_callback import ( + ApertureChangeCallback, +) from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( FGSCallbackCollection, ) @@ -50,24 +55,26 @@ class BlueskyRunner: command_queue: "Queue[Command]" = Queue() current_status: StatusAndMessage = StatusAndMessage(Status.IDLE) last_run_aborted: bool = False + aperture_change_callback = ApertureChangeCallback() def __init__(self, RE: RunEngine, skip_startup_connection=False) -> None: self.RE = RE self.skip_startup_connection = skip_startup_connection if VERBOSE_EVENT_LOGGING: RE.subscribe(VerbosePlanExecutionLoggingCallback()) + RE.subscribe(self.aperture_change_callback) if not self.skip_startup_connection: - for plan in PLAN_REGISTRY: - PLAN_REGISTRY[plan]["setup"]() + for plan_name in PLAN_REGISTRY: + PLAN_REGISTRY[plan_name]["setup"]() def start( - self, experiment: Callable, parameters: InternalParameters, plan: str + self, experiment: Callable, parameters: InternalParameters, plan_name: str ) -> StatusAndMessage: artemis.log.LOGGER.info(f"Started with parameters: {parameters}") if self.skip_startup_connection: - PLAN_REGISTRY[plan]["setup"]() + PLAN_REGISTRY[plan_name]["setup"]() self.callbacks = FGSCallbackCollection.from_params(parameters) if ( @@ -114,7 +121,11 @@ def wait_on_queue(self): try: with TRACER.start_span("do_run"): self.RE(command.experiment(command.parameters, self.callbacks)) - self.current_status = StatusAndMessage(Status.IDLE) + + self.current_status = StatusAndMessage( + Status.IDLE, + self.aperture_change_callback.last_selected_aperture, + ) self.last_run_aborted = False except WarningException as exception: artemis.log.LOGGER.warning("Warning Exception", exc_info=True) @@ -135,32 +146,51 @@ def __init__(self, runner: BlueskyRunner) -> None: super().__init__() self.runner = runner - def put(self, plan: str, action: Actions): + def put(self, plan_name: str, action: Actions): status_and_message = StatusAndMessage(Status.FAILED, f"{action} not understood") if action == Actions.START.value: try: - experiment_type = PLAN_REGISTRY.get(plan) - if experiment_type is None: + experiment_registry_entry = PLAN_REGISTRY.get(plan_name) + if experiment_registry_entry is None: + raise PlanNotFound( + f"Experiment plan '{plan_name}' not found in registry." + ) + + experiment_internal_param_type: InternalParameters = ( + experiment_registry_entry.get("internal_param_type") + ) + experiment = experiment_registry_entry.get("run") + if experiment_internal_param_type is None: raise PlanNotFound( - f"Experiment plan '{plan}' not found in registry." + f"Corresponding internal param type for '{plan_name}' not found in registry." ) - experiment = experiment_type.get("run") if experiment is None: raise PlanNotFound( - f"Experiment plan '{plan}' has no \"run\" method." + f"Experiment plan '{plan_name}' has no 'run' method." ) - parameters = InternalParameters.from_external_json(request.data) + parameters = experiment_internal_param_type.from_external_json( + request.data + ) status_and_message = self.runner.start( - experiment, parameters, experiment_type + experiment, parameters, plan_name ) except JSONDecodeError as e: status_and_message = StatusAndMessage(Status.FAILED, repr(e)) except PlanNotFound as e: status_and_message = StatusAndMessage(Status.FAILED, repr(e)) + except ValidationError as e: + status_and_message = StatusAndMessage(Status.FAILED, repr(e)) + artemis.log.LOGGER.error( + f" {format_exception(e)}: Invalid json parameters" + ) + except Exception as e: + status_and_message = StatusAndMessage(Status.FAILED, repr(e)) + artemis.log.LOGGER.error(format_exception(e)) + elif action == Actions.STOP.value: status_and_message = self.runner.stop() # no idea why mypy gives an attribute error here but nowhere else for this - # exactsame situation... + # exact same situation... return status_and_message.to_dict() # type: ignore @@ -193,7 +223,7 @@ def create_app( api = Api(app) api.add_resource( RunExperiment, - "//", + "//", resource_class_args=[runner], ) api.add_resource( @@ -225,7 +255,7 @@ def cli_arg_parse() -> ( help="Choose overall logging level, defaults to INFO", ) parser.add_argument( - "--skip_startup_connection", + "--skip-startup-connection", action="store_true", help="Skip connecting to EPICS PVs on startup", ) diff --git a/src/artemis/experiment_plans/experiment_registry.py b/src/artemis/experiment_plans/experiment_registry.py index 556390ba1..dd9e574ce 100644 --- a/src/artemis/experiment_plans/experiment_registry.py +++ b/src/artemis/experiment_plans/experiment_registry.py @@ -3,9 +3,15 @@ from typing import Callable, Dict, Union from dodal.devices.fast_grid_scan import GridScanParams -from dodal.devices.rotation_scan import RotationScanParams from artemis.experiment_plans import fast_grid_scan_plan +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.parameters.internal_parameters.plan_specific.rotation_scan_internal_params import ( + RotationInternalParameters, + RotationScanParams, +) def not_implemented(): @@ -21,16 +27,18 @@ def do_nothing(): "fast_grid_scan": { "setup": fast_grid_scan_plan.create_devices, "run": fast_grid_scan_plan.get_plan, - "param_type": GridScanParams, + "internal_param_type": FGSInternalParameters, + "experiment_param_type": GridScanParams, }, "rotation_scan": { "setup": do_nothing, "run": not_implemented, - "param_type": RotationScanParams, + "internal_param_type": RotationInternalParameters, + "experiment_param_type": RotationScanParams, }, } EXPERIMENT_NAMES = list(PLAN_REGISTRY.keys()) -EXPERIMENT_TYPE_LIST = [p["param_type"] for p in PLAN_REGISTRY.values()] +EXPERIMENT_TYPE_LIST = [p["experiment_param_type"] for p in PLAN_REGISTRY.values()] EXPERIMENT_TYPE_DICT = dict(zip(EXPERIMENT_NAMES, EXPERIMENT_TYPE_LIST)) diff --git a/src/artemis/experiment_plans/fast_grid_scan_plan.py b/src/artemis/experiment_plans/fast_grid_scan_plan.py old mode 100644 new mode 100755 index 3a28135fd..80f5201fb --- a/src/artemis/experiment_plans/fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/fast_grid_scan_plan.py @@ -30,6 +30,7 @@ setup_zebra_for_fgs, ) from artemis.exceptions import WarningException +from artemis.parameters import external_parameters from artemis.parameters.beamline_parameters import ( GDABeamlineParameters, get_beamline_prefixes, @@ -46,7 +47,9 @@ from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( FGSCallbackCollection, ) - from artemis.parameters.internal_parameters import InternalParameters + from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, + ) class FGSComposite: @@ -110,16 +113,24 @@ def set_aperture_for_bbox_size( bbox_size: list[int], ): # bbox_size is [x,y,z], for i03 we only care about x - if bbox_size[0] <= 1: - aperture_size_positions = aperture_device.aperture_positions.SMALL - elif 1 < bbox_size[0] < 3: + if bbox_size[0] < 2: aperture_size_positions = aperture_device.aperture_positions.MEDIUM + selected_aperture = "MEDIUM_APERTURE" else: aperture_size_positions = aperture_device.aperture_positions.LARGE + selected_aperture = "LARGE_APERTURE" artemis.log.LOGGER.info( - f"Setting aperture to {aperture_size_positions} based on bounding box size {bbox_size}." + f"Setting aperture to {selected_aperture} ({aperture_size_positions}) based on bounding box size {bbox_size}." + ) + + @bpp.set_run_key_decorator("change_aperture") + @bpp.run_decorator( + md={"subplan_name": "change_aperture", "aperture_size": selected_aperture} ) - yield from bps.abs_set(aperture_device, aperture_size_positions) + def set_aperture(): + yield from bps.abs_set(aperture_device, aperture_size_positions) + + yield from set_aperture() def read_hardware_for_ispyb( @@ -187,7 +198,7 @@ def tidy_up_plans(fgs_composite: FGSComposite): @bpp.run_decorator(md={"subplan_name": "run_gridscan"}) def run_gridscan( fgs_composite: FGSComposite, - parameters: InternalParameters, + parameters: FGSInternalParameters, md={ "plan_name": "run_gridscan", }, @@ -233,7 +244,7 @@ def do_fgs(): @bpp.run_decorator(md={"subplan_name": "run_gridscan_and_move"}) def run_gridscan_and_move( fgs_composite: FGSComposite, - parameters: InternalParameters, + parameters: FGSInternalParameters, subscriptions: FGSCallbackCollection, ): """A multi-run plan which runs a gridscan, gets the results from zocalo @@ -277,7 +288,7 @@ def gridscan_with_subscriptions(fgs_composite, params): def get_plan( - parameters: InternalParameters, + parameters: FGSInternalParameters, subscriptions: FGSCallbackCollection, ) -> Callable: """Create the plan to run the grid scan based on provided parameters. @@ -286,7 +297,7 @@ def get_plan( at any point in it. Args: - parameters (InternalParameters): The parameters to run the scan. + parameters (FGSInternalParameters): The parameters to run the scan. Returns: Generator: The plan for the gridscan @@ -317,8 +328,11 @@ def run_gridscan_and_move_and_tidy(fgs_composite, params, comms): RE = RunEngine({}) RE.waiting_hook = ProgressBarManager() + from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, + ) - parameters = InternalParameters(beamline=args.artemis_parameters.beamline) + parameters = FGSInternalParameters(external_parameters.from_file()) subscriptions = FGSCallbackCollection.from_params(parameters) create_devices() diff --git a/src/artemis/experiment_plans/tests/test_experiment_registry.py b/src/artemis/experiment_plans/tests/test_experiment_registry.py index 7145c42fe..47aeccafd 100644 --- a/src/artemis/experiment_plans/tests/test_experiment_registry.py +++ b/src/artemis/experiment_plans/tests/test_experiment_registry.py @@ -1,10 +1,15 @@ from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase from artemis.experiment_plans.experiment_registry import PLAN_REGISTRY +from artemis.parameters.internal_parameters import InternalParameters def test_experiment_registry_param_types(): for plan in PLAN_REGISTRY.keys(): assert issubclass( - PLAN_REGISTRY[plan]["param_type"], AbstractExperimentParameterBase + PLAN_REGISTRY[plan]["experiment_param_type"], + AbstractExperimentParameterBase, + ) + assert issubclass( + PLAN_REGISTRY[plan]["internal_param_type"], InternalParameters ) diff --git a/src/artemis/external_interaction/callbacks/aperture_change_callback.py b/src/artemis/external_interaction/callbacks/aperture_change_callback.py new file mode 100644 index 000000000..ab265a538 --- /dev/null +++ b/src/artemis/external_interaction/callbacks/aperture_change_callback.py @@ -0,0 +1,15 @@ +from bluesky.callbacks import CallbackBase + +from artemis.log import LOGGER + + +class ApertureChangeCallback(CallbackBase): + last_selected_aperture: str = "NONE" + + def start(self, doc: dict): + if doc.get("subplan_name") == "change_aperture": + LOGGER.info(f"START: {doc}") + ap_size = doc.get("aperture_size") + assert isinstance(ap_size, str) + LOGGER.info(f"Updating most recent in-plan aperture change to {ap_size}.") + self.last_selected_aperture = ap_size diff --git a/src/artemis/external_interaction/callbacks/fgs/ispyb_callback.py b/src/artemis/external_interaction/callbacks/fgs/ispyb_callback.py index 7ea7f5664..4b102cc16 100644 --- a/src/artemis/external_interaction/callbacks/fgs/ispyb_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/ispyb_callback.py @@ -13,7 +13,9 @@ ) from artemis.log import LOGGER, set_dcgid_tag from artemis.parameters.constants import ISPYB_PLAN_NAME, SIM_ISPYB_CONFIG -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) class FGSISPyBHandlerCallback(CallbackBase): @@ -33,7 +35,7 @@ class FGSISPyBHandlerCallback(CallbackBase): Usually used as part of an FGSCallbackCollection. """ - def __init__(self, parameters: InternalParameters): + def __init__(self, parameters: FGSInternalParameters): self.params = parameters self.descriptors: Dict[str, dict] = {} ispyb_config = os.environ.get("ISPYB_CONFIG_PATH", SIM_ISPYB_CONFIG) diff --git a/src/artemis/external_interaction/callbacks/fgs/nexus_callback.py b/src/artemis/external_interaction/callbacks/fgs/nexus_callback.py index c980f1fda..d215751f7 100644 --- a/src/artemis/external_interaction/callbacks/fgs/nexus_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/nexus_callback.py @@ -10,7 +10,9 @@ create_parameters_for_second_file, ) from artemis.log import LOGGER -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) class FGSNexusFileHandlerCallback(CallbackBase): @@ -30,7 +32,7 @@ class FGSNexusFileHandlerCallback(CallbackBase): Usually used as part of an FGSCallbackCollection. """ - def __init__(self, parameters: InternalParameters): + def __init__(self, parameters: FGSInternalParameters): self.nxs_writer_1 = NexusWriter(create_parameters_for_first_file(parameters)) self.nxs_writer_2 = NexusWriter(create_parameters_for_second_file(parameters)) self.run_gridscan_uid: Optional[str] = None diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py index 716af9ac1..e243e49c0 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py @@ -12,13 +12,16 @@ FGSCallbackCollection, ) from artemis.parameters.constants import SIM_BEAMLINE -from artemis.parameters.internal_parameters import InternalParameters from artemis.utils import create_point +from artemis.parameters.external_parameters import from_file as default_raw_params +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) def test_callback_collection_init(): - callbacks = FGSCallbackCollection.from_params(InternalParameters()) - test_parameters = InternalParameters() + test_parameters = FGSInternalParameters(default_raw_params()) + callbacks = FGSCallbackCollection.from_params(test_parameters) assert ( callbacks.ispyb_handler.params.experiment_params == test_parameters.experiment_params @@ -81,7 +84,7 @@ def test_communicator_in_composite_run( nexus_writer.side_effect = [MagicMock(), MagicMock()] RE = RunEngine({}) - params = InternalParameters() + params = FGSInternalParameters(default_raw_params()) params.artemis_params.beamline = SIM_BEAMLINE ispyb_begin_deposition.return_value = ([1, 2], None, 4) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_ispyb_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_ispyb_handler.py index 6cb6391b9..b9c58d73f 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_ispyb_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_ispyb_handler.py @@ -8,24 +8,32 @@ ) from artemis.external_interaction.callbacks.fgs.tests.conftest import TestData from artemis.log import LOGGER, set_up_logging_handlers -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.parameters.external_parameters import from_file as default_raw_params DC_IDS = [1, 2] DCG_ID = 4 td = TestData() +@pytest.fixture +def dummy_params(): + return FGSInternalParameters(default_raw_params()) + + def test_fgs_failing_results_in_bad_run_status_in_ispyb( mock_ispyb_update_time_and_status: MagicMock, mock_ispyb_get_time: MagicMock, mock_ispyb_store_grid_scan: MagicMock, + dummy_params, ): mock_ispyb_store_grid_scan.return_value = [DC_IDS, None, DCG_ID] mock_ispyb_get_time.return_value = td.DUMMY_TIME_STRING mock_ispyb_update_time_and_status.return_value = None - params = InternalParameters() - ispyb_handler = FGSISPyBHandlerCallback(params) + ispyb_handler = FGSISPyBHandlerCallback(dummy_params) ispyb_handler.start(td.test_start_document) ispyb_handler.descriptor(td.test_descriptor_document) ispyb_handler.event(td.test_event_document) @@ -49,13 +57,13 @@ def test_fgs_raising_no_exception_results_in_good_run_status_in_ispyb( mock_ispyb_update_time_and_status: MagicMock, mock_ispyb_get_time: MagicMock, mock_ispyb_store_grid_scan: MagicMock, + dummy_params, ): mock_ispyb_store_grid_scan.return_value = [DC_IDS, None, DCG_ID] mock_ispyb_get_time.return_value = td.DUMMY_TIME_STRING mock_ispyb_update_time_and_status.return_value = None - params = InternalParameters() - ispyb_handler = FGSISPyBHandlerCallback(params) + ispyb_handler = FGSISPyBHandlerCallback(dummy_params) ispyb_handler.start(td.test_start_document) ispyb_handler.descriptor(td.test_descriptor_document) ispyb_handler.event(td.test_event_document) @@ -83,12 +91,11 @@ def mock_emit(): def test_given_ispyb_callback_started_writing_to_ispyb_when_messages_logged_then_they_contain_dcgid( - mock_emit, mock_ispyb_store_grid_scan: MagicMock + mock_emit, mock_ispyb_store_grid_scan: MagicMock, dummy_params ): mock_ispyb_store_grid_scan.return_value = [DC_IDS, None, DCG_ID] - params = InternalParameters() - ispyb_handler = FGSISPyBHandlerCallback(params) + ispyb_handler = FGSISPyBHandlerCallback(dummy_params) ispyb_handler.start(td.test_start_document) ispyb_handler.descriptor(td.test_descriptor_document) @@ -105,13 +112,13 @@ def test_given_ispyb_callback_finished_writing_to_ispyb_when_messages_logged_the mock_ispyb_store_grid_scan: MagicMock, mock_ispyb_update_time_and_status: MagicMock, mock_ispyb_get_time: MagicMock, + dummy_params, ): mock_ispyb_store_grid_scan.return_value = [DC_IDS, None, DCG_ID] mock_ispyb_get_time.return_value = td.DUMMY_TIME_STRING mock_ispyb_update_time_and_status.return_value = None - params = InternalParameters() - ispyb_handler = FGSISPyBHandlerCallback(params) + ispyb_handler = FGSISPyBHandlerCallback(dummy_params) ispyb_handler.start(td.test_start_document) ispyb_handler.descriptor(td.test_descriptor_document) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_nexus_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_nexus_handler.py index 6eadbfe36..d359e5b77 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_nexus_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_nexus_handler.py @@ -6,7 +6,11 @@ from artemis.external_interaction.callbacks.fgs.nexus_callback import ( FGSNexusFileHandlerCallback, ) -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) + +from artemis.parameters.external_parameters import from_file as default_raw_params test_start_document = { "uid": "d8bee3ee-f614-4e7a-a516-25d6b9e87ef3", @@ -18,6 +22,11 @@ } +@pytest.fixture +def dummy_params(): + return FGSInternalParameters(default_raw_params()) + + @pytest.fixture def nexus_writer(): with patch( @@ -46,12 +55,11 @@ def test_writers_setup_on_init( params_for_second: MagicMock, params_for_first: MagicMock, nexus_writer: MagicMock, + dummy_params, ): - - params = InternalParameters() - nexus_handler = FGSNexusFileHandlerCallback(params) + nexus_handler = FGSNexusFileHandlerCallback(dummy_params) # flake8 gives an error if we don't do something with communicator - nexus_handler.__init__(params) + nexus_handler.__init__(dummy_params) nexus_writer.assert_has_calls( [ @@ -66,22 +74,20 @@ def test_writers_dont_create_on_init( params_for_second: MagicMock, params_for_first: MagicMock, nexus_writer: MagicMock, + dummy_params, ): - - params = InternalParameters() - nexus_handler = FGSNexusFileHandlerCallback(params) + nexus_handler = FGSNexusFileHandlerCallback(dummy_params) nexus_handler.nxs_writer_1.create_nexus_file.assert_not_called() nexus_handler.nxs_writer_2.create_nexus_file.assert_not_called() def test_writers_do_create_one_file_each_on_start_doc_for_run_gridscan( - nexus_writer: MagicMock, + nexus_writer: MagicMock, dummy_params ): nexus_writer.side_effect = [MagicMock(), MagicMock()] - params = InternalParameters() - nexus_handler = FGSNexusFileHandlerCallback(params) + nexus_handler = FGSNexusFileHandlerCallback(dummy_params) nexus_handler.start(test_start_document) nexus_handler.nxs_writer_1.create_nexus_file.assert_not_called() diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index 70f5a1375..8d3c585ab 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -9,8 +9,13 @@ from artemis.external_interaction.callbacks.fgs.tests.conftest import TestData from artemis.external_interaction.exceptions import ISPyBDepositionNotMade from artemis.external_interaction.zocalo.zocalo_interaction import NoDiffractionFound -from artemis.parameters.internal_parameters import InternalParameters from artemis.utils import create_point +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) + +from artemis.parameters.external_parameters import from_file as default_raw_params + EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -23,6 +28,11 @@ td = TestData() +@pytest.fixture +def dummy_params(): + return FGSInternalParameters(default_raw_params()) + + def mock_zocalo_functions(callbacks: FGSCallbackCollection): callbacks.zocalo_handler.zocalo_interactor.wait_for_result = MagicMock() callbacks.zocalo_handler.zocalo_interactor.run_end = MagicMock() @@ -34,6 +44,7 @@ def test_execution_of_run_gridscan_triggers_zocalo_calls( mock_ispyb_get_time: MagicMock, mock_ispyb_store_grid_scan: MagicMock, nexus_writer: MagicMock, + dummy_params, ): dc_ids = [1, 2] dcg_id = 4 @@ -42,8 +53,7 @@ def test_execution_of_run_gridscan_triggers_zocalo_calls( mock_ispyb_get_time.return_value = td.DUMMY_TIME_STRING mock_ispyb_update_time_and_status.return_value = None - params = InternalParameters() - callbacks = FGSCallbackCollection.from_params(params) + callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.start(td.test_run_gridscan_start_document) @@ -70,9 +80,10 @@ def test_execution_of_run_gridscan_triggers_zocalo_calls( callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_not_called() -def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_called(): - params = InternalParameters() - callbacks = FGSCallbackCollection.from_params(params) +def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_called( + dummy_params: FGSInternalParameters, +): + callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) expected_centre_grid_coords = create_point(1, 2, 3) @@ -93,7 +104,7 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal 100 ) expected_centre_motor_coords = ( - params.experiment_params.grid_position_to_motor_position( + dummy_params.experiment_params.grid_position_to_motor_position( create_point( expected_centre_grid_coords[0] - 0.5, expected_centre_grid_coords[1] - 0.5, @@ -104,9 +115,10 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal assert found_centre == expected_centre_motor_coords -def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_THEN_fallback_centre_used(): - params = InternalParameters() - callbacks = FGSCallbackCollection.from_params(params) +def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_THEN_fallback_centre_used( + dummy_params, +): + callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) callbacks.zocalo_handler.zocalo_interactor.wait_for_result.side_effect = ( @@ -122,18 +134,20 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ assert found_centre == fallback_position -def test_GIVEN_ispyb_not_started_WHEN_trigger_zocalo_handler_THEN_raises_exception(): - params = InternalParameters() - callbacks = FGSCallbackCollection.from_params(params) +def test_GIVEN_ispyb_not_started_WHEN_trigger_zocalo_handler_THEN_raises_exception( + dummy_params, +): + callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) with pytest.raises(ISPyBDepositionNotMade): callbacks.zocalo_handler.start(td.test_do_fgs_start_document) -def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_bbox_from_first(): - params = InternalParameters() - callbacks = FGSCallbackCollection.from_params(params) +def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_bbox_from_first( + dummy_params: FGSInternalParameters, +): + callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) expected_centre_grid_coords = create_point(4, 6, 2) @@ -163,7 +177,7 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b 100 ) expected_centre_motor_coords = ( - params.experiment_params.grid_position_to_motor_position( + dummy_params.experiment_params.grid_position_to_motor_position( create_point( expected_centre_grid_coords.x - 0.5, expected_centre_grid_coords.y - 0.5, diff --git a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py index 32d55e247..26794d466 100644 --- a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py @@ -16,8 +16,10 @@ ZocaloInteractor, ) from artemis.log import LOGGER -from artemis.parameters.internal_parameters import InternalParameters from artemis.utils import create_point +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) class FGSZocaloCallback(CallbackBase): @@ -41,7 +43,7 @@ class FGSZocaloCallback(CallbackBase): """ def __init__( - self, parameters: "InternalParameters", ispyb_handler: FGSISPyBHandlerCallback + self, parameters: FGSInternalParameters, ispyb_handler: FGSISPyBHandlerCallback ): self.grid_position_to_motor_position: Callable[ [ndarray], ndarray diff --git a/src/artemis/external_interaction/ispyb/store_in_ispyb.py b/src/artemis/external_interaction/ispyb/store_in_ispyb.py index c501f3d64..9e7435d3e 100755 --- a/src/artemis/external_interaction/ispyb/store_in_ispyb.py +++ b/src/artemis/external_interaction/ispyb/store_in_ispyb.py @@ -3,6 +3,7 @@ import datetime import re from abc import ABC, abstractmethod +from typing import TYPE_CHECKING import dodal.devices.oav.utils as oav_utils import ispyb @@ -11,10 +12,11 @@ from artemis.external_interaction.ispyb.ispyb_dataclass import Orientation from artemis.log import LOGGER -from artemis.parameters.internal_parameters import InternalParameters from artemis.tracing import TRACER from artemis.utils import create_point +if TYPE_CHECKING: + from artemis.parameters.internal_parameters import InternalParameters I03_EIGER_DETECTOR = 78 EIGER_FILE_SUFFIX = "h5" diff --git a/src/artemis/external_interaction/system_tests/conftest.py b/src/artemis/external_interaction/system_tests/conftest.py index 3feacf99b..39c667c3e 100644 --- a/src/artemis/external_interaction/system_tests/conftest.py +++ b/src/artemis/external_interaction/system_tests/conftest.py @@ -13,8 +13,11 @@ StoreInIspyb2D, StoreInIspyb3D, ) -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) from artemis.utils import create_point +from artemis.parameters.external_parameters import from_file as default_raw_params ISPYB_CONFIG = "/dls_sw/dasc/mariadb/credentials/ispyb-dev.cfg" @@ -75,7 +78,7 @@ def fetch_comment() -> Callable: @pytest.fixture def dummy_params(): - dummy_params = InternalParameters() + dummy_params = FGSInternalParameters(default_raw_params()) dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 diff --git a/src/artemis/external_interaction/system_tests/test_ispyb_dev_connection.py b/src/artemis/external_interaction/system_tests/test_ispyb_dev_connection.py index 9962aeab7..549630a55 100644 --- a/src/artemis/external_interaction/system_tests/test_ispyb_dev_connection.py +++ b/src/artemis/external_interaction/system_tests/test_ispyb_dev_connection.py @@ -5,7 +5,11 @@ StoreInIspyb2D, StoreInIspyb3D, ) -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.parameters.external_parameters import from_file as default_raw_params + ISPYB_CONFIG = "/dls_sw/dasc/mariadb/credentials/ispyb-dev.cfg" @@ -66,7 +70,7 @@ def test_ispyb_deposition_comment_correct_for_3D_on_failure( def test_can_store_2D_ispyb_data_correctly_when_in_error( StoreClass, exp_num_of_grids, success, fetch_comment ): - test_params = InternalParameters() + test_params = FGSInternalParameters(default_raw_params()) test_params.artemis_params.ispyb_params.visit_path = "/tmp/cm31105-4/" ispyb: StoreInIspyb = StoreClass(ISPYB_CONFIG, test_params) dc_ids, grid_ids, dcg_id = ispyb.begin_deposition() diff --git a/src/artemis/external_interaction/system_tests/test_zocalo_system.py b/src/artemis/external_interaction/system_tests/test_zocalo_system.py index e5d45ff98..e196fdf21 100644 --- a/src/artemis/external_interaction/system_tests/test_zocalo_system.py +++ b/src/artemis/external_interaction/system_tests/test_zocalo_system.py @@ -8,13 +8,17 @@ TEST_RESULT_LARGE, TEST_RESULT_SMALL, ) -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) from artemis.utils import create_point +from artemis.parameters.external_parameters import from_file as default_raw_params + @pytest.mark.s03 def test_when_running_start_stop_then_get_expected_returned_results(zocalo_env): - params = InternalParameters() + params = FGSInternalParameters(default_raw_params()) zc: FGSZocaloCallback = FGSCallbackCollection.from_params(params).zocalo_handler dcids = [1, 2] zc.ispyb.ispyb_ids = (dcids, 0, 4) @@ -27,7 +31,7 @@ def test_when_running_start_stop_then_get_expected_returned_results(zocalo_env): @pytest.fixture -def run_zocalo_with_dev_ispyb(dummy_params: InternalParameters, dummy_ispyb_3d): +def run_zocalo_with_dev_ispyb(dummy_params: FGSInternalParameters, dummy_ispyb_3d): def inner(sample_name="", fallback=create_point(0, 0, 0)): dummy_params.artemis_params.detector_params.prefix = sample_name zc: FGSZocaloCallback = FGSCallbackCollection.from_params( diff --git a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py index a1b3413d7..aaefa2f70 100644 --- a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py +++ b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py @@ -10,7 +10,10 @@ StoreInIspyb3D, ) from artemis.parameters.constants import SIM_ISPYB_CONFIG -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.parameters.external_parameters import from_file as default_raw_params from artemis.utils import create_point TEST_DATA_COLLECTION_IDS = [12, 13] @@ -24,7 +27,7 @@ @pytest.fixture def dummy_params(): - dummy_params = InternalParameters() + dummy_params = FGSInternalParameters(default_raw_params()) dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 @@ -96,7 +99,7 @@ def test_store_grid_scan(ispyb_conn, dummy_ispyb, dummy_params): @patch("ispyb.open", new_callable=mock_open) def test_store_3d_grid_scan( - ispyb_conn, dummy_ispyb_3d: StoreInIspyb3D, dummy_params: InternalParameters + ispyb_conn, dummy_ispyb_3d: StoreInIspyb3D, dummy_params: FGSInternalParameters ): ispyb_conn.return_value.mx_acquisition = mock() ispyb_conn.return_value.core = mock() @@ -224,7 +227,7 @@ def test_sample_id(default_params, actual): @patch("ispyb.open") def test_given_real_sampleid_when_grid_scan_stored_then_sample_id_set( - ispyb_conn, dummy_ispyb: StoreInIspyb2D, dummy_params: InternalParameters + ispyb_conn, dummy_ispyb: StoreInIspyb2D, dummy_params: FGSInternalParameters ): expected_sample_id = "0001" dummy_params.artemis_params.ispyb_params.sample_id = expected_sample_id @@ -350,7 +353,7 @@ def test_ispyb_deposition_comment_for_3D_correct( @patch("ispyb.open") def test_given_x_and_y_steps_different_from_total_images_when_grid_scan_stored_then_num_images_correct( - ispyb_conn, dummy_ispyb: StoreInIspyb2D, dummy_params: InternalParameters + ispyb_conn, dummy_ispyb: StoreInIspyb2D, dummy_params: FGSInternalParameters ): expected_number_of_steps = 200 * 3 dummy_params.experiment_params.x_steps = 200 diff --git a/src/artemis/external_interaction/unit_tests/test_write_nexus.py b/src/artemis/external_interaction/unit_tests/test_write_nexus.py index da5989255..b114f3981 100644 --- a/src/artemis/external_interaction/unit_tests/test_write_nexus.py +++ b/src/artemis/external_interaction/unit_tests/test_write_nexus.py @@ -12,13 +12,18 @@ create_parameters_for_first_file, create_parameters_for_second_file, ) -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) """It's hard to effectively unit test the nexus writing so these are really system tests that confirms that we're passing the right sorts of data to nexgen to get a sensible output. Note that the testing process does now write temporary files to disk.""" +from artemis.parameters.external_parameters import from_file as default_raw_params + + def assert_end_data_correct(nexus_writer: NexusWriter): for filename in [nexus_writer.nexus_file, nexus_writer.master_file]: with h5py.File(filename, "r") as written_nexus_file: @@ -27,7 +32,7 @@ def assert_end_data_correct(nexus_writer: NexusWriter): @pytest.fixture(params=[1044]) def minimal_params(request): - params = InternalParameters() + params = FGSInternalParameters(default_raw_params()) params.artemis_params.ispyb_params.wavelength = 1.0 params.artemis_params.ispyb_params.flux = 9.0 params.artemis_params.ispyb_params.transmission = 0.5 @@ -41,7 +46,7 @@ def minimal_params(request): @pytest.fixture -def dummy_nexus_writers(minimal_params: InternalParameters): +def dummy_nexus_writers(minimal_params: FGSInternalParameters): first_file_params = create_parameters_for_first_file(minimal_params) nexus_writer_1 = NexusWriter(first_file_params) @@ -56,7 +61,7 @@ def dummy_nexus_writers(minimal_params: InternalParameters): @pytest.fixture -def dummy_nexus_writers_with_more_images(minimal_params: InternalParameters): +def dummy_nexus_writers_with_more_images(minimal_params: FGSInternalParameters): x, y, z = 45, 35, 25 minimal_params.experiment_params.x_steps = x minimal_params.experiment_params.y_steps = y @@ -90,7 +95,7 @@ def single_dummy_file(minimal_params): indirect=["minimal_params"], ) def test_given_number_of_images_above_1000_then_expected_datafiles_used( - minimal_params: InternalParameters, expected_num_of_files, single_dummy_file + minimal_params: FGSInternalParameters, expected_num_of_files, single_dummy_file ): first_writer = single_dummy_file assert len(first_writer.get_image_datafiles()) == expected_num_of_files @@ -103,7 +108,7 @@ def test_given_number_of_images_above_1000_then_expected_datafiles_used( def test_given_dummy_data_then_datafile_written_correctly( - minimal_params: InternalParameters, + minimal_params: FGSInternalParameters, dummy_nexus_writers: tuple[NexusWriter, NexusWriter], ): nexus_writer_1, nexus_writer_2 = dummy_nexus_writers @@ -221,7 +226,7 @@ def assert_contains_external_link(data_path, entry_name, file_name): def test_nexus_writer_files_are_formatted_as_expected( - minimal_params: InternalParameters, single_dummy_file: NexusWriter + minimal_params: FGSInternalParameters, single_dummy_file: NexusWriter ): for file in [single_dummy_file.nexus_file, single_dummy_file.master_file]: file_name = os.path.basename(file.name) diff --git a/src/artemis/parameters/beamline_parameters.py b/src/artemis/parameters/beamline_parameters.py index 9c3ff5641..771abcfbd 100644 --- a/src/artemis/parameters/beamline_parameters.py +++ b/src/artemis/parameters/beamline_parameters.py @@ -33,10 +33,8 @@ def __getitem__(self, item: str): return self.params[item] @classmethod - def from_file(cls, path: str): + def from_lines(cls, config_lines: list[str]): ob = cls() - with open(path) as f: - config_lines = f.readlines() config_lines_nocomments = [line.split("#", 1)[0] for line in config_lines] config_lines_sep_key_and_value = [ line.translate(str.maketrans("", "", " \n\t\r")).split("=") @@ -58,3 +56,9 @@ def from_file(cls, path: str): config_pairs[i] = (config_pairs[i][0], float(config_pairs[i][1])) ob.params = dict(config_pairs) return ob + + @classmethod + def from_file(cls, path: str): + with open(path) as f: + config_lines = f.readlines() + return cls.from_lines(config_lines) diff --git a/src/artemis/parameters/constants.py b/src/artemis/parameters/constants.py index 54e4b549a..86154d19d 100644 --- a/src/artemis/parameters/constants.py +++ b/src/artemis/parameters/constants.py @@ -9,8 +9,8 @@ "/dls_sw/i03/software/daq_configuration/domain/beamlineParameters" ) PARAMETER_VERSION = 0.2 - SIM_ISPYB_CONFIG = "src/artemis/external_interaction/unit_tests/test_config.cfg" +PARAMETER_SCHEMA_DIRECTORY = "src/artemis/parameters/schemas/" DETECTOR_PARAM_DEFAULTS = { "current_energy": 100, diff --git a/src/artemis/parameters/external_parameters.py b/src/artemis/parameters/external_parameters.py index efa9461bb..47b0e8951 100644 --- a/src/artemis/parameters/external_parameters.py +++ b/src/artemis/parameters/external_parameters.py @@ -1,200 +1,35 @@ from __future__ import annotations -import copy import json -from dataclasses import dataclass, field +from os.path import join from pathlib import Path -from typing import Dict, Optional, Type, Union +from typing import Any import jsonschema -from dataclasses_json import DataClassJsonMixin -import artemis.experiment_plans.experiment_registry as registry -from artemis.parameters.constants import ( - PARAMETER_VERSION, - SIM_BEAMLINE, - SIM_INSERTION_PREFIX, - SIM_ZOCALO_ENV, -) +from artemis.parameters.constants import PARAMETER_SCHEMA_DIRECTORY -def default_field(obj): - return field(default_factory=lambda: copy.deepcopy(obj)) +def validate_raw_parameters_from_dict(dict_params: dict[str, Any]): + with open( + join(PARAMETER_SCHEMA_DIRECTORY, "full_external_parameters_schema.json"), "r" + ) as f: + full_schema = json.load(f) - -class WrongExperimentParameterSpecification(Exception): - pass - - -@dataclass -class ExternalDetectorParameters(DataClassJsonMixin): - current_energy: int = 100 - directory: str = "/tmp" - prefix: str = "file_name" - run_number: int = 0 - use_roi_mode: bool = False - det_dist_to_beam_converter_path: str = ( - "src/artemis/unit_tests/test_lookup_table.txt" - ) - detector_size_constants: Optional[str] = "EIGER2_X_16M" - - -@dataclass -class ExternalISPyBParameters(DataClassJsonMixin): - sample_id: Optional[int] = None - sample_barcode: Optional[str] = None - visit_path: str = "" - microns_per_pixel_x: float = 0.0 - microns_per_pixel_y: float = 0.0 - # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels - upper_left: Dict = default_field({"x": 0, "y": 0, "z": 0}) - position: Dict = default_field({"x": 0, "y": 0, "z": 0}) - xtal_snapshots_omega_start: list[str] = default_field( - ["test_1_y", "test_2_y", "test_3_y"] - ) - xtal_snapshots_omega_end: list[str] = default_field( - ["test_1_y", "test_2_y", "test_3_y"] - ) - transmission: float = 1.0 - flux: float = 10.0 - wavelength: float = 0.01 - beam_size_x: float = 0.1 - beam_size_y: float = 0.1 - focal_spot_size_x: float = 0.0 - focal_spot_size_y: float = 0.0 - comment: str = "Descriptive comment." - resolution: float = 1 - undulator_gap: float = 1.0 - synchrotron_mode: Optional[str] = None - slit_gap_size_x: float = 0.1 - slit_gap_size_y: float = 0.1 - - -@dataclass -class ExternalGridScanParameters(DataClassJsonMixin): - x_steps: int = 40 - y_steps: int = 20 - z_steps: int = 10 - x_step_size: float = 0.1 - y_step_size: float = 0.1 - z_step_size: float = 0.1 - dwell_time: float = 0.2 - x_start: float = 0.0 - y1_start: float = 0.0 - y2_start: float = 0.0 - z1_start: float = 0.0 - z2_start: float = 0.0 - exposure_time: float = 0.1 - detector_distance: float = 100.0 - omega_start: float = 0.0 - - -@dataclass -class ExternalRotationScanParameters(DataClassJsonMixin): - rotation_axis: str = "omega" - rotation_angle: float = 180.0 - x: float = 0.0 - y: float = 0.0 - z: float = 0.0 - omega_start: float = 0.0 - phi_start: float = 0.0 - chi_start: float = 0.0 - kappa_start: float = 0.0 - exposure_time: float = 0.1 - detector_distance: float = 100.0 - rotation_increment: float = 0.0 - - -@dataclass -class ExternalArtemisParameters(DataClassJsonMixin): - zocalo_environment: str = SIM_ZOCALO_ENV - beamline: str = SIM_BEAMLINE - insertion_prefix: str = SIM_INSERTION_PREFIX - experiment_type: str = registry.EXPERIMENT_NAMES[0] - detector_params: ExternalDetectorParameters = default_field( - ExternalDetectorParameters() + path = Path(PARAMETER_SCHEMA_DIRECTORY).absolute() + resolver = jsonschema.validators.RefResolver( + base_uri=f"{path.as_uri()}/", + referrer=True, ) - ispyb_params: ExternalISPyBParameters = default_field(ExternalISPyBParameters()) - - -EXTERNAL_EXPERIMENT_PARAM_TYPES = Union[ - ExternalGridScanParameters, ExternalRotationScanParameters -] -EXTERNAL_EXPERIMENT_PARAM_DICT: dict[str, Type] = { - "fast_grid_scan": ExternalGridScanParameters, - "rotation_scan": ExternalRotationScanParameters, -} - - -class RawParameters: - artemis_params: ExternalArtemisParameters - experiment_params: EXTERNAL_EXPERIMENT_PARAM_TYPES - - def __init__( - self, - artemis_parameters: ExternalArtemisParameters = ExternalArtemisParameters(), - experiment_parameters: EXTERNAL_EXPERIMENT_PARAM_TYPES = ExternalGridScanParameters(), - ) -> None: - self.artemis_params = copy.deepcopy(artemis_parameters) - self.experiment_params = copy.deepcopy(experiment_parameters) - - def __eq__(self, other) -> bool: - if not isinstance(other, RawParameters): - return NotImplemented - if self.artemis_params != other.artemis_params: - return False - if self.experiment_params != other.experiment_params: - return False - return True - - def to_dict(self) -> dict[str, dict]: - return { - "params_version": PARAMETER_VERSION, - "artemis_params": self.artemis_params.to_dict(), - "experiment_params": self.experiment_params.to_dict(), - } - - def to_json(self) -> str: - return json.dumps(self.to_dict()) + jsonschema.validate(dict_params, full_schema, resolver=resolver) + return dict_params - @classmethod - def from_dict(cls, dict_params: dict[str, dict]): - with open( - "src/artemis/parameters/schemas/full_external_parameters_schema.json", "r" - ) as f: - full_schema = json.load(f) - path = Path("src/artemis/parameters/schemas/").absolute() - resolver = jsonschema.validators.RefResolver( - base_uri=f"{path.as_uri()}/", - referrer=True, - ) - # TODO improve failed validation error messages - jsonschema.validate(dict_params, full_schema, resolver=resolver) - experiment_type = EXTERNAL_EXPERIMENT_PARAM_DICT.get( - dict_params["artemis_params"]["experiment_type"] - ) - try: - assert experiment_type is not None - experiment_params = experiment_type.from_dict( - dict_params["experiment_params"] - ) - except Exception: - raise WrongExperimentParameterSpecification( - "Either the experiment type parameter does not match a known experiment" - "type, or the experiment parameters were not correct." - ) - return cls( - ExternalArtemisParameters.from_dict(dict_params["artemis_params"]), - experiment_params, - ) +def from_json(json_params: str): + dict_params = json.loads(json_params) + return validate_raw_parameters_from_dict(dict_params) - @classmethod - def from_json(cls, json_params: str): - dict_params = json.loads(json_params) - return cls.from_dict(dict_params) - @classmethod - def from_file(cls, json_filename: str): - with open(json_filename) as f: - return cls.from_json(f.read()) +def from_file(json_filename: str = "test_parameter_defaults.json"): + with open(json_filename) as f: + return from_json(f.read()) diff --git a/src/artemis/parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters.py deleted file mode 100644 index bcbd01ae4..000000000 --- a/src/artemis/parameters/internal_parameters.py +++ /dev/null @@ -1,142 +0,0 @@ -from typing import Any, Dict - -from dodal.devices.eiger import DetectorParams, EigerTriggerNumber -from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase - -import artemis.experiment_plans.experiment_registry as registry -from artemis.external_interaction.ispyb.ispyb_dataclass import ( - ISPYB_PARAM_DEFAULTS, - IspybParams, -) -from artemis.parameters.constants import ( - DETECTOR_PARAM_DEFAULTS, - SIM_BEAMLINE, - SIM_INSERTION_PREFIX, - SIM_ZOCALO_ENV, -) -from artemis.parameters.external_parameters import RawParameters - - -class ArtemisParameters: - zocalo_environment: str = SIM_ZOCALO_ENV - beamline: str = SIM_BEAMLINE - insertion_prefix: str = SIM_INSERTION_PREFIX - experiment_type: str = registry.EXPERIMENT_NAMES[0] - - detector_params: DetectorParams = DetectorParams.from_dict(DETECTOR_PARAM_DEFAULTS) - ispyb_params: IspybParams = IspybParams.from_dict(ISPYB_PARAM_DEFAULTS) - - def __init__( - self, - zocalo_environment: str = SIM_ZOCALO_ENV, - beamline: str = SIM_BEAMLINE, - insertion_prefix: str = SIM_INSERTION_PREFIX, - experiment_type: str = registry.EXPERIMENT_NAMES[0], - detector_params: Dict[str, Any] = DETECTOR_PARAM_DEFAULTS, - ispyb_params: Dict[str, Any] = ISPYB_PARAM_DEFAULTS, - ) -> None: - self.zocalo_environment = zocalo_environment - self.beamline = beamline - self.insertion_prefix = insertion_prefix - self.experiment_type = experiment_type - self.detector_params: DetectorParams = DetectorParams.from_dict(detector_params) - self.ispyb_params: IspybParams = IspybParams.from_dict(ispyb_params) - - def __repr__(self): - r = "artemis_params:\n" - r += f" zocalo_environment: {self.zocalo_environment}\n" - r += f" beamline: {self.beamline}\n" - r += f" insertion_prefix: {self.insertion_prefix}\n" - r += f" experiment_type: {self.experiment_type}\n" - r += f" detector_params: {self.detector_params}\n" - r += f" ispyb_params: {self.ispyb_params}\n" - return r - - def __eq__(self, other) -> bool: - if not isinstance(other, ArtemisParameters): - return NotImplemented - elif self.zocalo_environment != other.zocalo_environment: - return False - elif self.beamline != other.beamline: - return False - elif self.insertion_prefix != other.insertion_prefix: - return False - elif self.experiment_type != other.experiment_type: - return False - elif self.detector_params != other.detector_params: - return False - elif self.ispyb_params != other.ispyb_params: - return False - return True - - -class InternalParameters: - artemis_params: ArtemisParameters - experiment_params: registry.EXPERIMENT_TYPES - - def __init__(self, external_params: RawParameters = RawParameters()): - ext_expt_param_dict = external_params.experiment_params.to_dict() - ext_art_param_dict = external_params.artemis_params.to_dict() - - rotation_inc = ext_expt_param_dict.get("rotation_increment") - if rotation_inc is None: - ext_art_param_dict["detector_params"]["omega_increment"] = 0 - else: - ext_art_param_dict["detector_params"][ - "omega_increment" - ] = ext_expt_param_dict["rotation_increment"] - - ext_art_param_dict["detector_params"]["omega_start"] = ext_expt_param_dict.pop( - "omega_start" - ) - - ext_art_param_dict["detector_params"][ - "detector_distance" - ] = ext_expt_param_dict.pop("detector_distance") - - ext_art_param_dict["detector_params"][ - "exposure_time" - ] = ext_expt_param_dict.pop("exposure_time") - - self.experiment_params: AbstractExperimentParameterBase = ( - registry.EXPERIMENT_TYPE_DICT[ext_art_param_dict["experiment_type"]]( - **ext_expt_param_dict - ) - ) - - n_images = self.experiment_params.get_num_images() - if self.experiment_params.trigger_number == EigerTriggerNumber.MANY_TRIGGERS: - ext_art_param_dict["detector_params"]["num_triggers"] = n_images - ext_art_param_dict["detector_params"]["num_images_per_trigger"] = 1 - else: - ext_art_param_dict["detector_params"]["num_triggers"] = 1 - ext_art_param_dict["detector_params"]["num_images_per_trigger"] = n_images - - self.artemis_params = ArtemisParameters(**ext_art_param_dict) - - def __repr__(self): - r = "[Artemis internal parameters]\n" - r += repr(self.artemis_params) - r += f"experiment_params: {self.experiment_params}" - return r - - def __eq__(self, other) -> bool: - if not isinstance(other, InternalParameters): - return NotImplemented - if self.artemis_params != other.artemis_params: - return False - if self.experiment_params != other.experiment_params: - return False - return True - - @classmethod - def from_external_json(cls, json_data): - """Convenience method to generate from external parameter JSON blob, uses - RawParameters.from_json()""" - return cls(RawParameters.from_json(json_data)) - - @classmethod - def from_external_dict(cls, dict_data): - """Convenience method to generate from external parameter dictionary, uses - RawParameters.from_dict()""" - return cls(RawParameters.from_dict(dict_data)) diff --git a/src/artemis/parameters/internal_parameters/__init__.py b/src/artemis/parameters/internal_parameters/__init__.py new file mode 100644 index 000000000..c4f056b65 --- /dev/null +++ b/src/artemis/parameters/internal_parameters/__init__.py @@ -0,0 +1,5 @@ +from artemis.parameters.internal_parameters.internal_parameters import ( + InternalParameters, +) + +__all__ = ["InternalParameters"] diff --git a/src/artemis/parameters/internal_parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters/internal_parameters.py new file mode 100644 index 000000000..6a6169a2a --- /dev/null +++ b/src/artemis/parameters/internal_parameters/internal_parameters.py @@ -0,0 +1,213 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict + +from dodal.devices.eiger import DetectorParams +from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase + +import artemis.parameters.external_parameters as raw_parameters +from artemis.external_interaction.ispyb.ispyb_dataclass import ( + ISPYB_PARAM_DEFAULTS, + IspybParams, +) +from artemis.parameters.constants import ( + DEFAULT_EXPERIMENT_TYPE, + DETECTOR_PARAM_DEFAULTS, + SIM_BEAMLINE, + SIM_INSERTION_PREFIX, + SIM_ZOCALO_ENV, +) +from artemis.utils import Point3D + + +class ArtemisParameters: + zocalo_environment: str = SIM_ZOCALO_ENV + beamline: str = SIM_BEAMLINE + insertion_prefix: str = SIM_INSERTION_PREFIX + experiment_type: str = DEFAULT_EXPERIMENT_TYPE + + detector_params: DetectorParams = DetectorParams.from_dict(DETECTOR_PARAM_DEFAULTS) + ispyb_params: IspybParams = IspybParams.from_dict(ISPYB_PARAM_DEFAULTS) + + def __init__( + self, + zocalo_environment: str = SIM_ZOCALO_ENV, + beamline: str = SIM_BEAMLINE, + insertion_prefix: str = SIM_INSERTION_PREFIX, + experiment_type: str = DEFAULT_EXPERIMENT_TYPE, + detector_params: Dict[str, Any] = DETECTOR_PARAM_DEFAULTS, + ispyb_params: Dict[str, Any] = ISPYB_PARAM_DEFAULTS, + ) -> None: + self.zocalo_environment = zocalo_environment + self.beamline = beamline + self.insertion_prefix = insertion_prefix + self.experiment_type = experiment_type + self.detector_params: DetectorParams = DetectorParams.from_dict(detector_params) + self.ispyb_params: IspybParams = IspybParams.from_dict(ispyb_params) + + def __repr__(self): + return ( + "artemis_params:\n" + f" zocalo_environment: {self.zocalo_environment}\n" + f" beamline: {self.beamline}\n" + f" insertion_prefix: {self.insertion_prefix}\n" + f" experiment_type: {self.experiment_type}\n" + f" detector_params: {self.detector_params}\n" + f" ispyb_params: {self.ispyb_params}\n" + ) + + def __eq__(self, other) -> bool: + if not isinstance(other, ArtemisParameters): + return NotImplemented + elif self.zocalo_environment != other.zocalo_environment: + return False + elif self.beamline != other.beamline: + return False + elif self.insertion_prefix != other.insertion_prefix: + return False + elif self.experiment_type != other.experiment_type: + return False + elif self.detector_params != other.detector_params: + return False + elif self.ispyb_params != other.ispyb_params: + return False + return True + + +def flatten_dict(d: dict, parent_items: dict = {}) -> dict: + """Flatten a dictionary assuming all keys are unique.""" + items: dict = {} + for k, v in d.items(): + if isinstance(v, dict): + flattened_subdict = flatten_dict(v, items) + items.update(flattened_subdict) + else: + if k in items or k in parent_items: + raise Exception(f"Duplicate keys '{k}' in input parameters!") + items[k] = v + return items + + +class InternalParameters(ABC): + """A base class with some helpful functions to aid in conversion from external + json parameters to internal experiment parameter classes, DetectorParams, + IspybParams, etc. + When subclassing you must provide the experiment parameter type as the + 'experiment_params_type' property, which must be a subclass of + dodal.parameters.experiment_parameter_base.AbstractExperimentParameterBase. + The corresponding initialisation values must be present in the external parameters + and be validated by the json schema. + Override or extend pre_sorting_translation() to modify key names or values before + sorting, and key_definitions() to determine which keys to send to DetectorParams and + IspybParams.""" + + artemis_params: ArtemisParameters + + def __init__(self, external_params: dict): + all_params_bucket = flatten_dict(external_params) + self.experiment_param_preprocessing(all_params_bucket) + + def fetch_subdict_from_bucket( + list_of_keys: list[str], bucket: dict[str, Any] + ) -> dict[str, Any]: + return { + key: bucket.get(key) + for key in list_of_keys + if bucket.get(key) is not None + } + + experiment_field_keys = list(self.experiment_params_type.__annotations__.keys()) + experiment_field_args: dict[str, Any] = fetch_subdict_from_bucket( + experiment_field_keys, all_params_bucket + ) + self.experiment_params: AbstractExperimentParameterBase = ( + self.experiment_params_type(**experiment_field_args) + ) + + self.artemis_param_preprocessing(all_params_bucket) + ( + artemis_param_field_keys, + detector_field_keys, + ispyb_field_keys, + ) = self.key_definitions() + + artemis_params_args: dict[str, Any] = fetch_subdict_from_bucket( + artemis_param_field_keys, all_params_bucket + ) + detector_params_args: dict[str, Any] = fetch_subdict_from_bucket( + detector_field_keys, all_params_bucket + ) + ispyb_params_args: dict[str, Any] = fetch_subdict_from_bucket( + ispyb_field_keys, all_params_bucket + ) + artemis_params_args["ispyb_params"] = ispyb_params_args + artemis_params_args["detector_params"] = detector_params_args + + self.artemis_params = ArtemisParameters(**artemis_params_args) + + @property + @abstractmethod + def experiment_params_type(self): + """This should be set to the experiment param type""" + pass + + def key_definitions(self): + artemis_param_field_keys = [ + "zocalo_environment", + "beamline", + "insertion_prefix", + "experiment_type", + ] + detector_field_keys = list(DetectorParams.__annotations__.keys()) + # not an annotation but specified as field encoder in DetectorParams: + detector_field_keys.append("detector") + ispyb_field_keys = list(IspybParams.__annotations__.keys()) + + return artemis_param_field_keys, detector_field_keys, ispyb_field_keys + + def experiment_param_preprocessing(self, param_dict: dict[str, Any]): + """operates on the supplied experiment parameter values befause the experiment + parameters object is initialised.""" + pass + + def artemis_param_preprocessing(self, param_dict: dict[str, Any]): + """Operates on the the flattened external param dictionary before its values are + distributed to the other dictionaries. In the default implementation, + self.experiment_params is already initialised, so values which are defined or + calculated there (e.g. num_images) are available. + Subclasses should extend or override this to define translations of names in the + external parameter set, applied to the param_dict. For example, in rotation + scans, `omega_increment` (for the detector) needs to come from the externally + supplied `rotation_increment` if the axis is omega. + """ + + param_dict["num_images"] = self.experiment_params.get_num_images() + param_dict["upper_left"] = Point3D(*param_dict["upper_left"]) + param_dict["position"] = Point3D(*param_dict["position"]) + + def __repr__(self): + return ( + "[Artemis internal parameters]\n" + f"{self.artemis_params}" + f"experiment_params: {self.experiment_params}" + ) + + def __eq__(self, other) -> bool: + if not isinstance(other, InternalParameters): + return NotImplemented + if self.artemis_params != other.artemis_params: + return False + if self.experiment_params != other.experiment_params: + return False + return True + + @classmethod + def from_external_json(cls, json_data): + """Convenience method to generate from external parameter JSON blob, uses + RawParameters.from_json()""" + return cls(raw_parameters.from_json(json_data)) + + @classmethod + def from_external_dict(cls, dict_data): + """Convenience method to generate from external parameter dictionary, uses + RawParameters.from_dict()""" + return cls(raw_parameters.validate_raw_parameters_from_dict(dict_data)) diff --git a/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py b/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py new file mode 100644 index 000000000..63be5605b --- /dev/null +++ b/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import Any + +from dodal.devices.fast_grid_scan import GridScanParams + +from artemis.parameters.internal_parameters import InternalParameters + + +class FGSInternalParameters(InternalParameters): + experiment_params_type = GridScanParams + experiment_params: GridScanParams + + def artemis_param_preprocessing(self, param_dict: dict[str, Any]): + super().artemis_param_preprocessing(param_dict) + param_dict["omega_increment"] = 0 + param_dict["num_triggers"] = param_dict["num_images"] + param_dict["num_images_per_trigger"] = 1 diff --git a/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py b/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py new file mode 100644 index 000000000..9a2bf05a3 --- /dev/null +++ b/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +from dataclasses_json import DataClassJsonMixin +from dodal.devices.eiger import EigerTriggerNumber +from dodal.devices.motors import XYZLimitBundle +from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase + +from artemis.parameters.internal_parameters import InternalParameters + + +@dataclass +class RotationScanParams(DataClassJsonMixin, AbstractExperimentParameterBase): + """ + Holder class for the parameters of a rotation data collection. + """ + + rotation_axis: str = "omega" + rotation_angle: float = 360.0 + image_width: float = 0.1 + omega_start: float = 0.0 + phi_start: float = 0.0 + chi_start: Optional[float] = None + kappa_start: Optional[float] = None + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + trigger_number: str = EigerTriggerNumber.MANY_TRIGGERS + rotation_direction: int = -1 + offset_deg: float = 1.0 + shutter_opening_time_s: float = 0.6 + + def xyz_are_valid(self, limits: XYZLimitBundle) -> bool: + """ + Validates scan location in x, y, and z + + :param limits: The motor limits against which to validate + the parameters + :return: True if the scan is valid + """ + if not limits.x.is_within(self.x): + return False + if not limits.y.is_within(self.y): + return False + if not limits.z.is_within(self.z): + return False + return True + + def get_num_images(self): + return int(self.rotation_angle / self.image_width) + + +class RotationInternalParameters(InternalParameters): + experiment_params_type = RotationScanParams + + def experiment_param_preprocessing(self, param_dict: dict[str, Any]): + positive_dir = param_dict.pop("positive_rotation_direction") + param_dict["rotation_direction"] = 1 if positive_dir else -1 + + def artemis_param_preprocessing(self, param_dict: dict[str, Any]): + super().artemis_param_preprocessing(param_dict) + if param_dict["rotation_axis"] == "omega": + param_dict["omega_increment"] = param_dict["rotation_increment"] + else: + param_dict["omega_increment"] = 0 + param_dict["num_triggers"] = 1 + param_dict["num_images_per_trigger"] = param_dict["num_images"] diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py new file mode 100644 index 000000000..3c3a52485 --- /dev/null +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py @@ -0,0 +1,26 @@ +from dodal.devices.det_dim_constants import EIGER2_X_16M_SIZE +from dodal.devices.fast_grid_scan import GridScanParams + +from artemis.parameters import external_parameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.utils import Point3D + + +def test_FGS_parameters_load_from_file(): + params = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_parameters.json" + ) + internal_parameters = FGSInternalParameters(params) + + assert isinstance(internal_parameters.experiment_params, GridScanParams) + + ispyb_params = internal_parameters.artemis_params.ispyb_params + + assert ispyb_params.position == Point3D(10, 20, 30) + assert ispyb_params.upper_left == Point3D(10, 20, 30) + + detector_params = internal_parameters.artemis_params.detector_params + + assert detector_params.detector_size_constants == EIGER2_X_16M_SIZE diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py new file mode 100644 index 000000000..6f42086fe --- /dev/null +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py @@ -0,0 +1,72 @@ +from unittest.mock import MagicMock + +from dodal.devices.det_dim_constants import EIGER2_X_16M_SIZE +from dodal.devices.motors import XYZLimitBundle + +from artemis.parameters import external_parameters +from artemis.parameters.internal_parameters.plan_specific.rotation_scan_internal_params import ( + RotationInternalParameters, + RotationScanParams, +) +from artemis.utils import Point3D + + +def test_rotation_scan_param_validity(): + test_params = RotationScanParams( + rotation_axis="omega", + rotation_angle=360, + image_width=0.1, + omega_start=0, + phi_start=0, + chi_start=0, + kappa_start=0, + x=0, + y=0, + z=0, + trigger_number="many_triggers", + ) + + xlim = MagicMock() + ylim = MagicMock() + zlim = MagicMock() + lims = XYZLimitBundle(xlim, ylim, zlim) + + assert test_params.xyz_are_valid(lims) + zlim.is_within.return_value = False + assert not test_params.xyz_are_valid(lims) + zlim.is_within.return_value = True + ylim.is_within.return_value = False + assert not test_params.xyz_are_valid(lims) + ylim.is_within.return_value = True + xlim.is_within.return_value = False + assert not test_params.xyz_are_valid(lims) + + +def test_rotation_parameters_load_from_file(): + params = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json" + ) + internal_parameters = RotationInternalParameters(params) + + assert isinstance(internal_parameters.experiment_params, RotationScanParams) + assert internal_parameters.experiment_params.rotation_direction == -1 + + ispyb_params = internal_parameters.artemis_params.ispyb_params + + assert ispyb_params.position == Point3D(10, 20, 30) + assert ispyb_params.upper_left == Point3D(10, 20, 30) + + detector_params = internal_parameters.artemis_params.detector_params + + assert detector_params.detector_size_constants == EIGER2_X_16M_SIZE + + +def test_rotation_parameters_preprocess(): + params = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json" + ) + params["experiment_params"]["positive_rotation_direction"] = True + internal_parameters = RotationInternalParameters(params) + assert isinstance(internal_parameters.experiment_params, RotationScanParams) + + assert internal_parameters.experiment_params.rotation_direction == 1 diff --git a/src/artemis/parameters/schemas/experiment_schemas/rotation_scan_params_schema.json b/src/artemis/parameters/schemas/experiment_schemas/rotation_scan_params_schema.json index 5c6511db4..66103347c 100644 --- a/src/artemis/parameters/schemas/experiment_schemas/rotation_scan_params_schema.json +++ b/src/artemis/parameters/schemas/experiment_schemas/rotation_scan_params_schema.json @@ -37,6 +37,15 @@ }, "z": { "type": "number" + }, + "positive_rotation_direction": { + "type": "boolean" + }, + "offset_deg": { + "type": "number" + }, + "shutter_opening_time_s": { + "type": "number" } }, "required": [ diff --git a/src/artemis/parameters/schemas/full_external_parameters_schema.json b/src/artemis/parameters/schemas/full_external_parameters_schema.json index fb9ff53e7..0166ddd65 100644 --- a/src/artemis/parameters/schemas/full_external_parameters_schema.json +++ b/src/artemis/parameters/schemas/full_external_parameters_schema.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "params_version": { - "const": 0.2 + "const": 0.3 }, "artemis_params": { "type": "object", diff --git a/src/artemis/parameters/schemas/ispyb_parameters_schema.json b/src/artemis/parameters/schemas/ispyb_parameters_schema.json index 625c52134..300140c36 100644 --- a/src/artemis/parameters/schemas/ispyb_parameters_schema.json +++ b/src/artemis/parameters/schemas/ispyb_parameters_schema.json @@ -12,74 +12,20 @@ "type": "number" }, "upper_left": { - "oneOf": [ - { - "type": "object", - "properties": { - "x": { - "type": [ - "number", - "null" - ] - }, - "y": { - "type": [ - "number", - "null" - ] - }, - "z": { - "type": [ - "number", - "null" - ] - } - } - }, - { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 3, - "maxItems": 3 - } - ] + "type": "array", + "items": { + "type": "number" + }, + "minItems": 3, + "maxItems": 3 }, "position": { - "oneOf": [ - { - "type": "object", - "properties": { - "x": { - "type": [ - "number", - "null" - ] - }, - "y": { - "type": [ - "number", - "null" - ] - }, - "z": { - "type": [ - "number", - "null" - ] - } - } - }, - { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 3, - "maxItems": 3 - } - ] + "type": "array", + "items": { + "type": "number" + }, + "minItems": 3, + "maxItems": 3 }, "xtal_snapshots_omega_start": { "type": "array", diff --git a/src/artemis/parameters/tests/test_data/good_test_parameters.json b/src/artemis/parameters/tests/test_data/good_test_parameters.json index 299fa8bb8..b933a2586 100644 --- a/src/artemis/parameters/tests/test_data/good_test_parameters.json +++ b/src/artemis/parameters/tests/test_data/good_test_parameters.json @@ -1,5 +1,5 @@ { - "params_version": 0.2, + "params_version": 0.3, "artemis_params": { "beamline": "BL03S", "detector": "EIGER2_X_16M", diff --git a/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json b/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json index 44e2196a3..288b1848c 100644 --- a/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json +++ b/src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json @@ -1,5 +1,5 @@ { - "params_version": 0.2, + "params_version": 0.3, "artemis_params": { "beamline": "BL03S", "detector": "EIGER2_X_16M", @@ -66,6 +66,9 @@ "z": 3.0, "exposure_time": 0.1, "detector_distance": 100.0, - "rotation_increment": 0.1 + "rotation_increment": 0.1, + "positive_rotation_direction": false, + "offset_deg": 1.0, + "shutter_opening_time_s": 0.6 } } \ No newline at end of file diff --git a/src/artemis/parameters/tests/test_data/test_beamline_parameters.txt b/src/artemis/parameters/tests/test_data/test_beamline_parameters.txt new file mode 100644 index 000000000..20e7eca27 --- /dev/null +++ b/src/artemis/parameters/tests/test_data/test_beamline_parameters.txt @@ -0,0 +1,299 @@ +# +# +BeamLine BL03I + +## BLSE=FB switches between scan alignment and feedback alignment +## by creating bl energy scannable with beamLineSpecificEnergy_FB +## after changing you must restart servers or >>> reset_namespace +BLSE=FB + +## BPFB (Beam Position FeedBack) +## HALF (default) only off during data collection +## FULL only off for XBPM2 during attenuation optimisation, fluo when trans < 2% and wedged MAD +## UNAVAILABLE (not default) prevents xbpm_feedback.py trying to access EPICS IOC that may not be running +BPFB=FULL +## Note: only beamline scientists control whether feedback is enabled +## via the XBPM feedback EDM screen in Synoptic + +# DCM parameters +DCM_Perp_Offset_FIXED = 25.6 +# +# beamstop +# +parked_x = 4.49 +parked_y = -50.0 +parked_y_plate = -50.5 +parked_z = -49.5 +parked_z_robot = 30.0 + +in_beam_z_MIN_START_POS = 60.0 + +in_beam_x_HIGHRES = 1.52 +in_beam_y_HIGHRES = 44.78 +in_beam_z_HIGHRES = 30.0 + +in_beam_x_STANDARD = 1.52 +in_beam_y_STANDARD = 44.78 +in_beam_z_STANDARD = 30.0 + +in_beam_x_LOWRES = 1.52 +in_beam_y_LOWRES = 44.78 +in_beam_z_LOWRES = 48 + +checkCryojet = No +#If is to be moved in by the script. If not Yes then control is handed to the robot on activate script +#To force the cryojet run hutch_utilities.hutch.forceCryoOut() +manualCryojet = Yes + +######################################################### +############# All these need checking! ############ +######################################################### + +#Aperture - Scatterguard positions +# 100 micron ap +miniap_x_LARGE_APERTURE = 2.389 +miniap_y_LARGE_APERTURE = 40.986 +miniap_z_LARGE_APERTURE = 15.8 + +sg_x_LARGE_APERTURE = 5.25 +sg_y_LARGE_APERTURE = 4.43 + +# 50 micron ap +miniap_x_MEDIUM_APERTURE = 2.384 +miniap_y_MEDIUM_APERTURE = 44.967 +miniap_z_MEDIUM_APERTURE = 15.8 +sg_x_MEDIUM_APERTURE = 5.285 +sg_y_MEDIUM_APERTURE = 0.46 + +# 20 micron ap +miniap_x_SMALL_APERTURE = 2.430 +miniap_y_SMALL_APERTURE = 48.974 +miniap_z_SMALL_APERTURE = 15.8 +sg_x_SMALL_APERTURE = 5.3375 +sg_y_SMALL_APERTURE = -3.55 + +# Robot load +miniap_x_ROBOT_LOAD = 2.386 +miniap_y_ROBOT_LOAD = 31.40 +miniap_z_ROBOT_LOAD = 15.8 +sg_x_ROBOT_LOAD = 5.25 +sg_y_ROBOT_LOAD = 4.43 + +# manual mount +miniap_x_MANUAL_LOAD = -4.91 +miniap_y_MANUAL_LOAD = -49.0 +miniap_z_MANUAL_LOAD = -10.0 + +sg_x_MANUAL_LOAD = -4.7 +sg_y_MANUAL_LOAD = 1.8 + +miniap_x_SCIN_MOVE = -4.91 +# prion setting +#miniap_x_SCIN_MOVE = 0.0 +sg_x_SCIN_MOVE = -4.75 + +scin_y_SCIN_IN = 100.855 +scin_y_SCIN_OUT = -0.02 +scin_z_SCIN_IN = 101.5115 + + +scin_z_SCIN_OUT = 0.1 + +#distance to move gonx,y,z when scintillator is put in with standard pins +# For old gonio: +gon_x_SCIN_OUT_DISTANCE = 1.0 +# For SmarGon: +gon_x_SCIN_OUT_DISTANCE_smargon = 1 + +gon_y_SCIN_OUT_DISTANCE = 2.0 +gon_z_SCIN_OUT_DISTANCE = -0.5 + +#CASS motor position tolerances (mm) +miniap_x_tolerance = 0.004 +miniap_y_tolerance = 0.1 +miniap_z_tolerance = 0.1 +sg_x_tolerance = 0.1 +sg_y_tolerance = 0.1 +scin_y_tolerance = 0.1 +scin_z_tolerance = 0.12 +gon_x_tolerance = 0.01 +gon_y_tolerance = 0.1 +gon_z_tolerance = 0.001 +bs_x_tolerance = 0.02 +bs_y_tolerance = 0.005 +bs_z_tolerance = 0.3 +crl_x_tolerance = 0.01 +crl_y_tolerance = 0.01 +crl_pitch_tolerance = 0.01 +crl_yaw_tolerance = 0.01 +sg_y_up_movement_tolerance = 1.0 + +sg_x_timeout = 10 +sg_y_timeout = 10 +miniap_x_timeout = 60 +miniap_y_timeout = 10 +gon_x_timeout = 60 +gon_y_timeout = 30 +gon_z_timeout = 30 +crl_x_timeout = 10 +crl_y_timeout = 10 +crl_pitch_timeout = 10 +crl_yaw_timeout = 10 + +col_inbeam_tolerance = 1.0 + +# robot load collimation table reference positions (mm) +col_parked_tolerance = 1.0 +col_parked_upstream_x = 0.0 +col_parked_downstream_x = 0.0 +col_parked_upstream_y = 0.0 +col_parked_inboard_y = 0.0 +col_parked_outboard_y = 0.0 + +## CRL positions for low and high energy lens sets. Should deliver beam to same position on scintillator. +## Normally should only adjust the low energy set to match the position of the high energy that you've +## already checked on the scintillator screen. + +crl_x_LOWE = -11.78 +crl_y_LOWE = -4.3 +crl_pitch_LOWE = -4.75 +crl_yaw_LOWE = -1.0 + +crl_x_HIGHE = 2.22 +crl_y_HIGHE = -4.30 +crl_pitch_HIGHE = -2.75 +crl_yaw_HIGHE = 0 + + +######################################################### +########## End of new parameters ########### +######################################################### + + +#Beam visualisation parameters +MinBackStopZ = 30.0 +BackStopYsafe = 20.0 +BackStopXyag = -4.8 +BackStopYyag = 17.20 +BackStopZyag = 19.1 +SampleYnormal = 2.65 +SampleYshift = 2.0 +parked_fluo_x = -18.0 +in_beam_fluo_x = 12.0 +move_fluo = Yes +safe_det_z_default = 900 +safe_det_z_sampleChanger = 337 +store_data_collections_in_ispyb = Yes +TakePNGsOfSample = Yes + +#robot requires these values +gonio_parked_x = 0.0 +gonio_parked_y = 0.0 +gonio_parked_z = 0.0 +gonio_parked_omega = 0 +gonio_parked_chi = 0 +gonio_parked_phi = 0 + +# The following used by setupBeamLine script +setupBeamLine_energyStart = 7000.0 +setupBeamLine_energyEnd = 17000.0 +setupBeamLine_energyStep = 500 +setupBeamLine_rollStart = -4 +setupBeamLine_rollEnd = 4 +setupBeamLine_rollSteps = 21 +setupBeamLine_pitchStart = -3.7 +setupBeamLine_pitchEnd = -3.5 +setupBeamLine_pitchSteps = 200 +#values below in microns +beamXCentre = 0 +beamYCentre = 0 +beamXYSettleTime = 6.0 +beamXYTolerance = 5.0 +DataCollection_TurboMode = Yes +#time in seconds. If not set then the default is 0.1 + +#The following are used by beamLineenergy script +beamLineEnergy_rollBeamX 50 +beamLineEnergy_rollBeamY 200 +beamLineEnergy__rollWidth = .2 +beamLineEnergy__rollStep = .02 +beamLineEnergy__pitchWidth = .02 +beamLineEnergy__pitchStep = .002 +beamLineEnergy__fpitchWidth = .02 +beamLineEnergy__fpitchStep = .001 +beamLineEnergy__adjustSlits = No +#dataCollectionMinSampleCurrent = 0.245 +dataCollectionMinSampleCurrent = 0.000 +dataCollectionSampleCurrent qbpm3 + +#Mark is using the following in some test scripts +MinIPin = 1.0 +YAGPin = 1 +RotationAxisPin = 2 +PtPin = 3 +PowderPin = 4 + +iPinInDetZ = 340.0 + +DataCollectionDetX = -7.8504 +DataCollectionDetYaw = 6.499 +DataCollectionDetY = 48.0 + +# StandardEnergy on i03 is 12700eV +StandardEnergy = 12700 + +keyence_max_attempts = 1 +# Move gonio 100 microns, see difference in keyence values +# Then do 100/difference, put that number below +# Sign may change between Smargon and MiniKappa +keyence_slopeYToX = 2.5 +keyence_slopeYToY = -2.5 +keyence_slopeXToZ = 3.23 + +YAGSamX = 1022 +YAGSamY = -98.0 +YAGSamZ = -147 +YAGOmega = 0.0 + +#ipin value must be < ipin_threshold above background for data collection +ipin_threshold = 0.1 + +# energy thresholds for mirror stripes +# - first threshold is between bare/Rh stripes (e.g. 7000) +# - second threshold is between Rh/Pt stripes (e.g. 18000) +mirror_threshold_bare_rh = 6900 +mirror_threshold_rh_pt = 30000 + +# flux conversion factors +flux_factor_no_aperture = 1 +flux_factor_LARGE_APERTURE = 0.738 +flux_factor_MEDIUM_APERTURE = 0.36 +flux_factor_SMALL_APERTURE = 0.084 +flux_factor_no_aperture_plate = 1 +flux_factor_LARGE_APERTURE_plate = 0.738 +flux_factor_MEDIUM_APERTURE_plate = 0.36 +flux_factor_SMALL_APERTURE_plate = 0.084 + +# assuming gain 10^3 +pin_diode_factor = 2.66E19 + +# Fluorescence/Vortex detector settings +attenuation_optimisation_type = deadtime # deadtime or total_counts + +#Deadtime settings +fluorescence_analyser_deadtimeThreshold=0.002 # used by edge scans +fluorescence_spectrum_deadtimeThreshold=0.0005 # used by spectrum + +#Other settings +fluorescence_attenuation_low_roi = 100 +fluorescence_attenuation_high_roi = 2048 +attenuation_optimisation_optimisation_cycles = 10 +attenuation_optimisation_start_transmission = 0.1 # per cent +fluorescence_mca_sca_offset = 400 + +#Total count settings +attenuation_optimisation_multiplier = 2 +attenuation_optimisation_target_count = 2000 +attenuation_optimisation_upper_limit = 50000 +attenuation_optimisation_lower_limit = 20000 + diff --git a/src/artemis/parameters/tests/test_external_parameters.py b/src/artemis/parameters/tests/test_external_parameters.py index a7f4bcdfc..a8e43f8e1 100644 --- a/src/artemis/parameters/tests/test_external_parameters.py +++ b/src/artemis/parameters/tests/test_external_parameters.py @@ -1,87 +1,43 @@ -import json +from artemis.parameters import external_parameters +from artemis.parameters.beamline_parameters import GDABeamlineParameters -from pytest import raises -from artemis.parameters.external_parameters import ( - ExternalGridScanParameters, - ExternalRotationScanParameters, - RawParameters, - WrongExperimentParameterSpecification, -) - - -def test_new_parameters_is_a_deep_copy(): - first_copy = RawParameters() - second_copy = RawParameters() - assert first_copy == second_copy - assert first_copy is not second_copy - assert ( - first_copy.artemis_params.detector_params - is not second_copy.artemis_params.detector_params +def test_new_parameters_is_a_new_object(): + a = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_parameters.json" ) - assert first_copy.experiment_params is not second_copy.experiment_params - assert ( - first_copy.artemis_params.ispyb_params - is not second_copy.artemis_params.ispyb_params + b = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_parameters.json" ) + assert a == b + assert a is not b def test_parameters_load_from_file(): - params = RawParameters.from_file( + params = external_parameters.from_file( "src/artemis/parameters/tests/test_data/good_test_parameters.json" ) - expt_params: ExternalGridScanParameters = params.experiment_params - assert isinstance(expt_params, ExternalGridScanParameters) - assert expt_params.x_steps == 5 - assert expt_params.y_steps == 10 - assert expt_params.z_steps == 2 - assert expt_params.x_step_size == 0.1 - assert expt_params.y_step_size == 0.1 - assert expt_params.z_step_size == 0.1 - assert expt_params.dwell_time == 0.2 - assert expt_params.x_start == 0.0 - assert expt_params.y1_start == 0.0 - assert expt_params.y2_start == 0.0 - assert expt_params.z1_start == 0.0 - assert expt_params.z2_start == 0.0 - - params = RawParameters.from_file( - "src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json" + expt_params = params["experiment_params"] + assert expt_params["x_steps"] == 5 + assert expt_params["y_steps"] == 10 + assert expt_params["z_steps"] == 2 + assert expt_params["x_step_size"] == 0.1 + assert expt_params["y_step_size"] == 0.1 + assert expt_params["z_step_size"] == 0.1 + assert expt_params["dwell_time"] == 0.2 + assert expt_params["x_start"] == 0.0 + assert expt_params["y1_start"] == 0.0 + assert expt_params["y2_start"] == 0.0 + assert expt_params["z1_start"] == 0.0 + assert expt_params["z2_start"] == 0.0 + + +def test_beamline_parameters(): + params = GDABeamlineParameters.from_file( + "src/artemis/parameters/tests/test_data/test_beamline_parameters.txt" ) - expt_params: ExternalRotationScanParameters = params.experiment_params - assert isinstance(params.experiment_params, ExternalRotationScanParameters) - assert expt_params.rotation_axis == "omega" - assert expt_params.rotation_angle == 180.0 - assert expt_params.omega_start == 0.0 - assert expt_params.phi_start == 0.0 - assert expt_params.chi_start == 0 - assert expt_params.x == 1.0 - assert expt_params.y == 2.0 - assert expt_params.z == 3.0 - - -def test_parameter_eq(): - params = RawParameters() - - assert not params == 6 - assert not params == "" - - params2 = RawParameters() - assert params == params2 - params2.artemis_params.insertion_prefix = "" - assert not params == params2 - - params2 = RawParameters() - assert params == params2 - params2.experiment_params.x_start = 12345 - assert not params == params2 - - -def test_parameter_init_with_bad_type_raises_exception(): - with open( - "src/artemis/parameters/tests/test_data/good_test_rotation_scan_parameters.json" - ) as f: - param_dict = json.load(f) - param_dict["artemis_params"]["experiment_type"] = "nonsense_scan" - with raises(WrongExperimentParameterSpecification): - params = RawParameters.from_dict(param_dict) # noqa: F841 + assert params["sg_x_MEDIUM_APERTURE"] == 5.285 + assert params["col_parked_downstream_x"] == 0 + assert params["beamLineEnergy__pitchStep"] == 0.002 + assert params["DataCollection_TurboMode"] is True + assert params["beamLineEnergy__adjustSlits"] is False diff --git a/src/artemis/parameters/tests/test_internal_parameters.py b/src/artemis/parameters/tests/test_internal_parameters.py index a3457fe26..7a2abddad 100644 --- a/src/artemis/parameters/tests/test_internal_parameters.py +++ b/src/artemis/parameters/tests/test_internal_parameters.py @@ -1,25 +1,187 @@ -from dodal.devices.det_dim_constants import EIGER2_X_16M_SIZE -from dodal.devices.fast_grid_scan import GridScanParams +import copy +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch -from artemis.parameters.external_parameters import RawParameters -from artemis.parameters.internal_parameters import InternalParameters +import pytest +from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase +from artemis.parameters import external_parameters +from artemis.parameters.internal_parameters.internal_parameters import ( + InternalParameters, + flatten_dict, +) +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) from artemis.utils import create_point +TEST_PARAM_DICT = { + "layer_1": { + "a": 23, + "x": 56, + "k": 47, + "layer_2": {"l": 11, "h": "test_value"}, + "layer_3": {"b": 5, "c": 6, "y": 7, "z": "test_value_2"}, + } +} -def test_parameters_load_from_file(): - params = RawParameters.from_file( +TEST_TRANSFORMED_PARAM_DICT: dict[str, Any] = { + "a": 23, + "b": 5, + "c": 6, + "detector_params": {"x": 56, "y": 7, "z": "test_value_2"}, + "ispyb_params": {"h": "test_value", "k": 47, "l": 11}, +} + +TEST_TRANSFORMED_PARAM_DICT_2: dict[str, Any] = { + "a": 23, + "b": 5, + "c": 6, + "detector_params": {"x": 56, "y": 7, "z": "test_value_2"}, + "ispyb_params": {"h": "test_value", "k": 47, "q": 11}, +} + + +class ParamTypeForTesting(AbstractExperimentParameterBase): + trigger_number = "many_triggers" + + def get_num_images(self): + return 15 + + +class InternalParametersSubclassForTesting(InternalParameters): + def artemis_param_preprocessing(self, param_dict: dict[str, Any]): + pass + + def key_definitions(self): + artemis_params = ["a", "b", "c"] + detector_params = ["x", "y", "z"] + ispyb_params = ["h", "k", "l"] + return artemis_params, detector_params, ispyb_params + + experiment_params_type = ParamTypeForTesting + + +class InternalParametersSubclass2(InternalParameters): + def artemis_param_preprocessing(self, param_dict: dict[str, Any]): + param_dict["q"] = param_dict["l"] + + def key_definitions(self): + artemis_params = ["a", "b", "c"] + detector_params = ["x", "y", "z"] + ispyb_params = ["h", "k", "q"] + return artemis_params, detector_params, ispyb_params + + experiment_params_type = ParamTypeForTesting + + +@dataclass +class FakeArtemisParams: + a: int + b: int + c: int + detector_params: MagicMock + ispyb_params: MagicMock + + +def test_cant_initialise_abstract_internalparams(): + with pytest.raises(TypeError): + internal_parameters = InternalParameters( # noqa + external_parameters.from_file() + ) + + +@patch( + "artemis.parameters.internal_parameters.internal_parameters.ArtemisParameters", + FakeArtemisParams, +) +@patch("artemis.parameters.internal_parameters.internal_parameters.DetectorParams") +@patch("artemis.parameters.internal_parameters.internal_parameters.IspybParams") +def test_initialise_and_verify_transformation( + ispybparams: MagicMock, detectorparams: MagicMock +): + test_params = InternalParametersSubclassForTesting(TEST_PARAM_DICT) + assert test_params.artemis_params.a == TEST_TRANSFORMED_PARAM_DICT["a"] + assert test_params.artemis_params.b == TEST_TRANSFORMED_PARAM_DICT["b"] + assert test_params.artemis_params.c == TEST_TRANSFORMED_PARAM_DICT["c"] + test_params.artemis_params.detector_params == ( + TEST_TRANSFORMED_PARAM_DICT["detector_params"] + ) + test_params.artemis_params.ispyb_params == ( + TEST_TRANSFORMED_PARAM_DICT["ispyb_params"] + ) + + +@patch( + "artemis.parameters.internal_parameters.internal_parameters.ArtemisParameters", + FakeArtemisParams, +) +@patch("artemis.parameters.internal_parameters.internal_parameters.DetectorParams") +@patch("artemis.parameters.internal_parameters.internal_parameters.IspybParams") +def test_pre_sorting_transformation(ispybparams: MagicMock, detectorparams: MagicMock): + test_params = InternalParametersSubclass2(TEST_PARAM_DICT) + assert test_params.artemis_params.a == TEST_TRANSFORMED_PARAM_DICT_2["a"] + assert test_params.artemis_params.b == TEST_TRANSFORMED_PARAM_DICT_2["b"] + assert test_params.artemis_params.c == TEST_TRANSFORMED_PARAM_DICT_2["c"] + test_params.artemis_params.detector_params == ( + TEST_TRANSFORMED_PARAM_DICT_2["detector_params"] + ) + test_params.artemis_params.ispyb_params == ( + TEST_TRANSFORMED_PARAM_DICT_2["ispyb_params"] + ) + + +def test_flatten(): + params = external_parameters.from_file( "src/artemis/parameters/tests/test_data/good_test_parameters.json" ) - internal_parameters = InternalParameters(params) + flat_dict = flatten_dict(params) + for k in flat_dict: + assert not isinstance(flat_dict[k], dict) + + flat_test_dict = flatten_dict(TEST_PARAM_DICT) + for k in ["a", "b", "c", "h", "k", "l", "x", "y", "z"]: + assert k in flat_test_dict + + with pytest.raises(Exception): + flatten_dict({"x": 6, "y": {"x": 7}}) + + +def test_internal_params_eq(): + params = external_parameters.from_file( + "src/artemis/parameters/tests/test_data/good_test_parameters.json" + ) + internal_params = FGSInternalParameters(params) + internal_params_2 = copy.deepcopy(internal_params) + + assert internal_params == internal_params_2 + assert internal_params_2 != 3 + assert internal_params_2.artemis_params != 3 + + internal_params_2.experiment_params.x_steps = 11111 + assert internal_params != internal_params_2 + + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.ispyb_params.beam_size_x = 123456 + assert internal_params != internal_params_2 - assert isinstance(internal_parameters.experiment_params, GridScanParams) + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.detector_params.exposure_time = 99999 + assert internal_params != internal_params_2 - ispyb_params = internal_parameters.artemis_params.ispyb_params + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.zocalo_environment = "not_real_env" + assert internal_params != internal_params_2 - assert ispyb_params.position == create_point(10, 20, 30) - assert ispyb_params.upper_left == create_point(10, 20, 30) + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.beamline = "not_real_beamline" + assert internal_params != internal_params_2 - detector_params = internal_parameters.artemis_params.detector_params + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.insertion_prefix = "not_real_prefix" + assert internal_params != internal_params_2 - assert detector_params.detector_size_constants == EIGER2_X_16M_SIZE + internal_params_2 = copy.deepcopy(internal_params) + internal_params_2.artemis_params.experiment_type = "not_real_experiment" + assert internal_params != internal_params_2 diff --git a/src/artemis/system_tests/test_aperturescatterguard_system.py b/src/artemis/system_tests/test_aperturescatterguard_system.py index 673f3d7b6..b83eebbb9 100644 --- a/src/artemis/system_tests/test_aperturescatterguard_system.py +++ b/src/artemis/system_tests/test_aperturescatterguard_system.py @@ -1,12 +1,5 @@ -import bluesky.plan_stubs as bps import pytest -from bluesky.callbacks import CallbackBase -from bluesky.run_engine import RunEngine -from dodal.devices.aperturescatterguard import ( - AperturePositions, - ApertureScatterguard, - InvalidApertureMove, -) +from dodal.devices.aperturescatterguard import AperturePositions, ApertureScatterguard from artemis.parameters.beamline_parameters import GDABeamlineParameters from artemis.parameters.constants import I03_BEAMLINE_PARAMETER_PATH @@ -23,124 +16,18 @@ def ap_sg(): return ap_sg -@pytest.fixture -def move_to_large(ap_sg: ApertureScatterguard): - yield from bps.abs_set(ap_sg, ap_sg.aperture_positions.LARGE) - - -@pytest.fixture -def move_to_medium(ap_sg: ApertureScatterguard): - yield from bps.abs_set(ap_sg, ap_sg.aperture_positions.MEDIUM) - - -@pytest.fixture -def move_to_small(ap_sg: ApertureScatterguard): - yield from bps.abs_set(ap_sg, ap_sg.aperture_positions.SMALL) - - -@pytest.fixture -def move_to_robotload(ap_sg: ApertureScatterguard): - yield from bps.abs_set(ap_sg, ap_sg.aperture_positions.ROBOT_LOAD) - - -@pytest.mark.s03 -def test_aperturescatterguard_setup(ap_sg: ApertureScatterguard): - ap_sg.wait_for_connection() - assert ap_sg.aperture_positions is not None - - -@pytest.mark.s03 -def test_aperturescatterguard_move_in_plan( - ap_sg: ApertureScatterguard, - move_to_large, - move_to_medium, - move_to_small, - move_to_robotload, -): - RE = RunEngine({}) - ap_sg.wait_for_connection() - - ap_sg.aperture.z.set(ap_sg.aperture_positions.LARGE[2], wait=True) - - RE(move_to_large) - RE(move_to_medium) - RE(move_to_small) - RE(move_to_robotload) +@pytest.mark.s03() +def test_aperture_change_callback(ap_sg: ApertureScatterguard): + from bluesky.run_engine import RunEngine + from artemis.experiment_plans.fast_grid_scan_plan import set_aperture_for_bbox_size + from artemis.external_interaction.callbacks.aperture_change_callback import ( + ApertureChangeCallback, + ) -@pytest.mark.s03 -def test_move_fails_when_not_in_good_starting_pos( - ap_sg: ApertureScatterguard, move_to_large -): - RE = RunEngine({}) ap_sg.wait_for_connection() - - ap_sg.aperture.z.set(0, wait=True) - - with pytest.raises(InvalidApertureMove): - RE(move_to_large) - - -class MonitorCallback(CallbackBase): - # holds on to the most recent time a motor move completed for aperture and - # scatterguard y - - t_ap_y: float = 0 - t_sg_y: float = 0 - event_docs: list[dict] = [] - - def event(self, doc): - self.event_docs.append(doc) - if doc["data"].get("ap_sg_aperture_y_motor_done_move") == 1: - self.t_ap_y = doc["timestamps"].get("ap_sg_aperture_y_motor_done_move") - if doc["data"].get("ap_sg_scatterguard_y_motor_done_move") == 1: - self.t_sg_y = doc["timestamps"].get("ap_sg_scatterguard_y_motor_done_move") - - -@pytest.mark.s03 -@pytest.mark.parametrize( - "pos1,pos2,sg_first", - [ - ("L", "M", True), - ("L", "S", True), - ("L", "R", False), - ("M", "L", False), - ("M", "S", True), - ("M", "R", False), - ("S", "L", False), - ("S", "M", False), - ("S", "R", False), - ("R", "L", True), - ("R", "M", True), - ("R", "S", True), - ], -) -def test_aperturescatterguard_moves_in_correct_order( - pos1, pos2, sg_first, ap_sg: ApertureScatterguard -): - cb = MonitorCallback() - positions = { - "L": ap_sg.aperture_positions.LARGE, - "M": ap_sg.aperture_positions.MEDIUM, - "S": ap_sg.aperture_positions.SMALL, - "R": ap_sg.aperture_positions.ROBOT_LOAD, - } - pos1 = positions[pos1] - pos2 = positions[pos2] + cb = ApertureChangeCallback() RE = RunEngine({}) RE.subscribe(cb) - - ap_sg.wait_for_connection() - ap_sg.aperture.z.set(pos1[2], wait=True) - - def monitor_and_moves(): - yield from bps.open_run() - yield from bps.monitor(ap_sg.aperture.y.motor_done_move, name="ap_y") - yield from bps.monitor(ap_sg.scatterguard.y.motor_done_move, name="sg_y") - yield from bps.mv(ap_sg, pos1) - yield from bps.mv(ap_sg, pos2) - yield from bps.close_run() - - RE(monitor_and_moves()) - - assert (cb.t_sg_y < cb.t_ap_y) == sg_first + RE(set_aperture_for_bbox_size(ap_sg, [2, 2, 2])) + assert cb.last_selected_aperture == "LARGE_APERTURE" diff --git a/src/artemis/system_tests/test_device_setups_and_cleanups.py b/src/artemis/system_tests/test_device_setups_and_cleanups.py index 817263e17..c584a2813 100644 --- a/src/artemis/system_tests/test_device_setups_and_cleanups.py +++ b/src/artemis/system_tests/test_device_setups_and_cleanups.py @@ -23,25 +23,29 @@ def RE(): return RunEngine({}) -@pytest.mark.s03 -def test_zebra_set_up_for_fgs(RE): +@pytest.fixture +def connected_zebra(): zebra = Zebra(name="zebra", prefix="BL03S-EA-ZEBRA-01:") - RE(setup_zebra_for_fgs(zebra)) - assert zebra.output.out_pvs[TTL_DETECTOR].get() == IN3_TTL - assert zebra.output.out_pvs[TTL_SHUTTER].get() == IN4_TTL + zebra.wait_for_connection() + return zebra @pytest.mark.s03 -def test_zebra_set_up_for_rotation(RE): - zebra = Zebra(name="zebra", prefix="BL03S-EA-ZEBRA-01:") - RE(setup_zebra_for_rotation(zebra)) - assert zebra.pc.gate_trigger.get(as_string=True) == I03_axes.OMEGA.value - assert zebra.pc.gate_width.get() == pytest.approx(360, 0.01) +def test_zebra_set_up_for_fgs(RE, connected_zebra: Zebra): + RE(setup_zebra_for_fgs(connected_zebra)) + assert connected_zebra.output.out_pvs[TTL_DETECTOR].get() == IN3_TTL + assert connected_zebra.output.out_pvs[TTL_SHUTTER].get() == IN4_TTL @pytest.mark.s03 -def test_zebra_cleanup(RE): - zebra = Zebra(name="zebra", prefix="BL03S-EA-ZEBRA-01:") - RE(set_zebra_shutter_to_manual(zebra)) - assert zebra.output.out_pvs[TTL_DETECTOR].get() == PC_PULSE - assert zebra.output.out_pvs[TTL_SHUTTER].get() == OR1 +def test_zebra_set_up_for_rotation(RE, connected_zebra: Zebra): + RE(setup_zebra_for_rotation(connected_zebra)) + assert connected_zebra.pc.gate_trigger.get(as_string=True) == I03_axes.OMEGA.value + assert connected_zebra.pc.gate_width.get() == pytest.approx(360, 0.01) + + +@pytest.mark.s03 +def test_zebra_cleanup(RE, connected_zebra: Zebra): + RE(set_zebra_shutter_to_manual(connected_zebra)) + assert connected_zebra.output.out_pvs[TTL_DETECTOR].get() == PC_PULSE + assert connected_zebra.output.out_pvs[TTL_SHUTTER].get() == OR1 diff --git a/src/artemis/system_tests/test_fgs_plan.py b/src/artemis/system_tests/test_fgs_plan.py index a5349a6c1..f40cee1aa 100644 --- a/src/artemis/system_tests/test_fgs_plan.py +++ b/src/artemis/system_tests/test_fgs_plan.py @@ -29,10 +29,17 @@ ) from artemis.parameters.beamline_parameters import GDABeamlineParameters from artemis.parameters.constants import I03_BEAMLINE_PARAMETER_PATH, SIM_BEAMLINE -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) +from artemis.parameters.external_parameters import from_file as default_raw_params + -params = InternalParameters() -params.artemis_params.beamline = SIM_BEAMLINE +@pytest.fixture +def params(): + params = FGSInternalParameters(default_raw_params()) + params.artemis_params.beamline = SIM_BEAMLINE + return params @pytest.fixture @@ -83,15 +90,16 @@ def fgs_composite(): @pytest.mark.skip(reason="Broken due to eiger issues in s03") @pytest.mark.s03 -@patch("artemis.fast_grid_scan_plan.wait_for_fgs_valid") @patch("bluesky.plan_stubs.wait") @patch("bluesky.plan_stubs.kickoff") @patch("bluesky.plan_stubs.complete") +@patch("artemis.fast_grid_scan_plan.wait_for_fgs_valid") def test_run_gridscan( wait_for_fgs_valid: MagicMock, complete: MagicMock, kickoff: MagicMock, wait: MagicMock, + params: FGSInternalParameters, RE: RunEngine, fgs_composite: FGSComposite, ): @@ -130,9 +138,10 @@ def test_full_plan_tidies_at_end( kickoff: MagicMock, wait: MagicMock, fgs_composite: FGSComposite, + params: FGSInternalParameters, RE: RunEngine, ): - callbacks = FGSCallbackCollection.from_params(InternalParameters()) + callbacks = FGSCallbackCollection.from_params(params) RE(get_plan(params, callbacks)) set_shutter_to_manual.assert_called_once() @@ -151,9 +160,10 @@ def test_full_plan_tidies_at_end_when_plan_fails( kickoff: MagicMock, wait: MagicMock, fgs_composite: FGSComposite, + params: FGSInternalParameters, RE: RunEngine, ): - callbacks = FGSCallbackCollection.from_params(InternalParameters()) + callbacks = FGSCallbackCollection.from_params(params) run_gridscan_and_move.side_effect = Exception() with pytest.raises(Exception): RE(get_plan(params, callbacks)) @@ -162,11 +172,9 @@ def test_full_plan_tidies_at_end_when_plan_fails( @pytest.mark.s03 def test_GIVEN_scan_invalid_WHEN_plan_run_THEN_ispyb_entry_made_but_no_zocalo_entry( - RE: RunEngine, - fgs_composite: FGSComposite, - fetch_comment: Callable, + RE: RunEngine, fgs_composite: FGSComposite, fetch_comment: Callable, params ): - parameters = InternalParameters() + parameters = FGSInternalParameters(params) parameters.artemis_params.detector_params.directory = "./tmp" parameters.artemis_params.detector_params.prefix = str(uuid.uuid1()) parameters.artemis_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" @@ -199,25 +207,25 @@ def test_WHEN_plan_run_THEN_move_to_centre_returned_from_zocalo_expected_centre( RE: RunEngine, fgs_composite: FGSComposite, zocalo_env: None, + params, ): """This test currently avoids hardware interaction and is mostly confirming interaction with dev_ispyb and dev_zocalo""" - parameters = InternalParameters() - parameters.artemis_params.detector_params.directory = "./tmp" - parameters.artemis_params.detector_params.prefix = str(uuid.uuid1()) - parameters.artemis_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" + params.artemis_params.detector_params.directory = "./tmp" + params.artemis_params.detector_params.prefix = str(uuid.uuid1()) + params.artemis_params.ispyb_params.visit_path = "/dls/i03/data/2022/cm31105-5/" # Currently s03 calls anything with z_steps > 1 invalid - parameters.experiment_params.z_steps = 1 + params.experiment_params.z_steps = 1 fgs_composite.eiger.stage = MagicMock() fgs_composite.eiger.unstage = MagicMock() - callbacks = FGSCallbackCollection.from_params(parameters) + callbacks = FGSCallbackCollection.from_params(params) callbacks.ispyb_handler.ispyb.ISPYB_CONFIG_PATH = ISPYB_CONFIG - RE(get_plan(parameters, callbacks)) + RE(get_plan(params, callbacks)) # The following numbers are derived from the centre returned in fake_zocalo assert fgs_composite.sample_motors.x.user_readback.get() == pytest.approx(0.05) diff --git a/src/artemis/system_tests/test_main_system.py b/src/artemis/system_tests/test_main_system.py index cb4a1d91b..5fcb6dad4 100644 --- a/src/artemis/system_tests/test_main_system.py +++ b/src/artemis/system_tests/test_main_system.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import threading from dataclasses import dataclass @@ -11,14 +13,18 @@ from artemis.__main__ import Actions, BlueskyRunner, Status, cli_arg_parse, create_app from artemis.experiment_plans.experiment_registry import PLAN_REGISTRY -from artemis.parameters.external_parameters import RawParameters +from artemis.parameters import external_parameters +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) FGS_ENDPOINT = "/fast_grid_scan/" START_ENDPOINT = FGS_ENDPOINT + Actions.START.value STOP_ENDPOINT = Actions.STOP.value STATUS_ENDPOINT = Actions.STATUS.value SHUTDOWN_ENDPOINT = Actions.SHUTDOWN.value -TEST_PARAMS = RawParameters().to_json() +TEST_PARAMS = json.dumps(external_parameters.from_file("test_parameters.json")) +TEST_BAD_PARAM_ENDPOINT = "/fgs_real_params/" + Actions.START.value class MockRunEngine: @@ -39,6 +45,9 @@ def abort(self): raise Exception(self.error) self.RE_takes_time = False + def subscribe(self, *args): + pass + @dataclass class ClientAndRunEngine: @@ -50,12 +59,38 @@ def mock_dict_values(d: dict): return {k: MagicMock() for k, _ in d.items()} +TEST_EXPTS = { + "test_experiment": { + "setup": MagicMock(), + "run": MagicMock(), + "internal_param_type": MagicMock(), + "experiment_param_type": MagicMock(), + }, + "test_experiment_no_run": { + "setup": MagicMock(), + "internal_param_type": MagicMock(), + "experiment_param_type": MagicMock(), + }, + "test_experiment_no_internal_param_type": { + "setup": MagicMock(), + "run": MagicMock(), + "experiment_param_type": MagicMock(), + }, + "fgs_real_params": { + "setup": MagicMock(), + "run": MagicMock(), + "internal_param_type": FGSInternalParameters, + "experiment_param_type": MagicMock(), + }, +} + + @pytest.fixture def test_env(): mock_run_engine = MockRunEngine() with patch.dict( "artemis.__main__.PLAN_REGISTRY", - {k: mock_dict_values(v) for k, v in PLAN_REGISTRY.items()}, + dict({k: mock_dict_values(v) for k, v in PLAN_REGISTRY.items()}, **TEST_EXPTS), ): app, runner = create_app({"TESTING": True}, mock_run_engine) runner_thread = threading.Thread(target=runner.wait_on_queue) @@ -63,7 +98,9 @@ def test_env(): with app.test_client() as client: with patch.dict( "artemis.__main__.PLAN_REGISTRY", - {k: mock_dict_values(v) for k, v in PLAN_REGISTRY.items()}, + dict( + {k: mock_dict_values(v) for k, v in PLAN_REGISTRY.items()}, **TEST_EXPTS + ), ): yield ClientAndRunEngine(client, mock_run_engine) @@ -120,6 +157,30 @@ def test_putting_bad_plan_fails(test_env: ClientAndRunEngine): ) +def test_plan_with_no_params_fails(test_env: ClientAndRunEngine): + response = test_env.client.put( + "/test_experiment_no_internal_param_type/start", data=TEST_PARAMS + ).json + assert isinstance(response, dict) + assert response.get("status") == Status.FAILED.value + assert ( + response.get("message") + == "PlanNotFound(\"Corresponding internal param type for 'test_experiment_no_internal_param_type' not found in registry.\")" + ) + + +def test_plan_with_no_run_fails(test_env: ClientAndRunEngine): + response = test_env.client.put( + "/test_experiment_no_run/start", data=TEST_PARAMS + ).json + assert isinstance(response, dict) + assert response.get("status") == Status.FAILED.value + assert ( + response.get("message") + == "PlanNotFound(\"Experiment plan 'test_experiment_no_run' has no 'run' method.\")" + ) + + def test_sending_start_twice_fails(test_env: ClientAndRunEngine): test_env.client.put(START_ENDPOINT, data=TEST_PARAMS) response = test_env.client.put(START_ENDPOINT, data=TEST_PARAMS) @@ -206,7 +267,7 @@ def test_cli_args_parse(): "--dev", "--logging-level=DEBUG", "--verbose-event-logging", - "--skip_startup_connection", + "--skip-startup-connection", ] test_args = cli_arg_parse() assert test_args == ("DEBUG", True, True, True) @@ -311,3 +372,14 @@ def test_when_blueskyrunner_initiated_and_skip_flag_is_not_set_then_all_plans_se ): BlueskyRunner(MagicMock(), skip_startup_connection=False) assert mock_setup.call_count == 3 + + +def test_log_on_invalid_json_params(caplog, test_env: ClientAndRunEngine): + response = test_env.client.put(TEST_BAD_PARAM_ENDPOINT, data='{"bad":1}').json + assert isinstance(response, dict) + assert response.get("status") == Status.FAILED.value + assert ( + response.get("message") + == "" + ) + assert "Invalid json parameters" in caplog.text diff --git a/src/artemis/unit_tests/test_fast_grid_scan_plan.py b/src/artemis/unit_tests/test_fast_grid_scan_plan.py index b2b5800a0..3777b0673 100644 --- a/src/artemis/unit_tests/test_fast_grid_scan_plan.py +++ b/src/artemis/unit_tests/test_fast_grid_scan_plan.py @@ -38,14 +38,20 @@ TEST_RESULT_SMALL, ) from artemis.log import set_up_logging_handlers -from artemis.parameters.external_parameters import RawParameters -from artemis.parameters.internal_parameters import InternalParameters +from artemis.parameters import external_parameters +from artemis.parameters.internal_parameters.internal_parameters import ( + InternalParameters, +) +from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( + FGSInternalParameters, +) from artemis.utils import create_point +from artemis.parameters.external_parameters import from_file as default_raw_params @pytest.fixture def test_params(): - return InternalParameters() + return FGSInternalParameters(default_raw_params()) @pytest.fixture @@ -95,18 +101,18 @@ def mock_subscriptions(test_params): return subscriptions -def test_given_full_parameters_dict_when_detector_name_used_and_converted_then_detector_constants_correct(): - params = InternalParameters(RawParameters()) +def test_given_full_parameters_dict_when_detector_name_used_and_converted_then_detector_constants_correct( + test_params: FGSInternalParameters, +): assert ( - params.artemis_params.detector_params.detector_size_constants.det_type_string + test_params.artemis_params.detector_params.detector_size_constants.det_type_string == EIGER_TYPE_EIGER2_X_16M ) - raw_params_dict = RawParameters().to_dict() + raw_params_dict = external_parameters.from_file() raw_params_dict["artemis_params"]["detector_params"][ "detector_size_constants" ] = EIGER_TYPE_EIGER2_X_4M - raw_params = RawParameters.from_dict(raw_params_dict) - params: InternalParameters = InternalParameters(raw_params) + params: FGSInternalParameters = FGSInternalParameters(raw_params_dict) det_dimension = ( params.artemis_params.detector_params.detector_size_constants.det_dimension ) @@ -119,10 +125,9 @@ def test_when_run_gridscan_called_then_generator_returned(): def test_read_hardware_for_ispyb_updates_from_ophyd_devices( - fake_fgs_composite: FGSComposite, + fake_fgs_composite: FGSComposite, test_params: FGSInternalParameters ): RE = RunEngine({}) - params = InternalParameters() undulator_test_value = 1.234 @@ -138,7 +143,7 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( fake_fgs_composite.s4_slit_gaps.xgap.user_readback.sim_put(xgap_test_value) fake_fgs_composite.s4_slit_gaps.ygap.user_readback.sim_put(ygap_test_value) - test_ispyb_callback = FGSISPyBHandlerCallback(params) + test_ispyb_callback = FGSISPyBHandlerCallback(test_params) test_ispyb_callback.ispyb = MagicMock() RE.subscribe(test_ispyb_callback) @@ -216,12 +221,9 @@ def test_results_adjusted_and_passed_to_move_xyz( call_medium = call( *(fake_fgs_composite.aperture_scatterguard.aperture_positions.MEDIUM) ) - call_small = call( - *(fake_fgs_composite.aperture_scatterguard.aperture_positions.SMALL) - ) move_aperture.assert_has_calls( - [call_large, call_medium, call_small], any_order=True + [call_large, call_large, call_medium], any_order=True ) @@ -260,12 +262,11 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( move_aperture: MagicMock, mock_subscriptions: FGSCallbackCollection, fake_fgs_composite: FGSComposite, - test_params: FGSComposite, + test_params: FGSInternalParameters, ): RE = RunEngine({}) set_up_logging_handlers(logging_level="INFO", dev_mode=True) RE.subscribe(VerbosePlanExecutionLoggingCallback()) - params = InternalParameters() RE( run_gridscan_and_move( @@ -275,7 +276,7 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( ) ) - run_gridscan.assert_called_once_with(fake_fgs_composite, params) + run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] np.testing.assert_array_almost_equal( array_arg, create_point(0.05, 0.15000000000000002, 0.25) diff --git a/test_parameter_defaults.json b/test_parameter_defaults.json new file mode 100644 index 000000000..36c83a9d3 --- /dev/null +++ b/test_parameter_defaults.json @@ -0,0 +1,74 @@ +{ + "params_version": 0.3, + "artemis_params": { + "zocalo_environment": "dev_artemis", + "beamline": "BL03S", + "insertion_prefix": "SR03S", + "experiment_type": "fast_grid_scan", + "detector_params": { + "current_energy": 100, + "directory": "/tmp/", + "prefix": "file_name", + "run_number": 0, + "use_roi_mode": false, + "det_dist_to_beam_converter_path": "src/artemis/unit_tests/test_lookup_table.txt" + }, + "ispyb_params": { + "visit_path": "", + "microns_per_pixel_x": 0.0, + "microns_per_pixel_y": 0.0, + "upper_left": [ + 0, + 0, + 0 + ], + "position": [ + 0, + 0, + 0 + ], + "xtal_snapshots_omega_start": [ + "test_1_y", + "test_2_y", + "test_3_y" + ], + "xtal_snapshots_omega_end": [ + "test_1_z", + "test_2_z", + "test_3_z" + ], + "transmission": 1.0, + "flux": 10.0, + "wavelength": 0.01, + "beam_size_x": 0.1, + "beam_size_y": 0.1, + "focal_spot_size_x": 0.0, + "focal_spot_size_y": 0.0, + "comment": "Descriptive comment.", + "resolution": 1, + "sample_id": null, + "sample_barcode": null, + "undulator_gap": 1.0, + "synchrotron_mode": null, + "slit_gap_size_x": 0.1, + "slit_gap_size_y": 0.1 + } + }, + "experiment_params": { + "x_steps": 40, + "y_steps": 20, + "z_steps": 10, + "x_step_size": 0.1, + "y_step_size": 0.1, + "z_step_size": 0.1, + "dwell_time": 0.2, + "x_start": 0.0, + "y1_start": 0.0, + "y2_start": 0.0, + "z1_start": 0.0, + "z2_start": 0.0, + "detector_distance": 100.0, + "omega_start": 0.0, + "exposure_time": 0.1 + } +} \ No newline at end of file diff --git a/test_parameters.json b/test_parameters.json index 299fa8bb8..b933a2586 100644 --- a/test_parameters.json +++ b/test_parameters.json @@ -1,5 +1,5 @@ { - "params_version": 0.2, + "params_version": 0.3, "artemis_params": { "beamline": "BL03S", "detector": "EIGER2_X_16M", From 0c045513547555e3d91455cce9d3711be8e79c78 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Thu, 20 Apr 2023 10:23:12 +0100 Subject: [PATCH 08/22] minor fixes --- .../parameters/internal_parameters/internal_parameters.py | 6 +++--- .../plan_specific/tests/test_fgs_internal_parameters.py | 6 +++--- .../tests/test_rotation_internal_parameters.py | 6 +++--- src/artemis/parameters/tests/test_internal_parameters.py | 1 - 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/artemis/parameters/internal_parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters/internal_parameters.py index 6a6169a2a..e20da9a85 100644 --- a/src/artemis/parameters/internal_parameters/internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/internal_parameters.py @@ -16,7 +16,7 @@ SIM_INSERTION_PREFIX, SIM_ZOCALO_ENV, ) -from artemis.utils import Point3D +from artemis.utils import create_point class ArtemisParameters: @@ -181,8 +181,8 @@ def artemis_param_preprocessing(self, param_dict: dict[str, Any]): """ param_dict["num_images"] = self.experiment_params.get_num_images() - param_dict["upper_left"] = Point3D(*param_dict["upper_left"]) - param_dict["position"] = Point3D(*param_dict["position"]) + param_dict["upper_left"] = create_point(*param_dict["upper_left"]) + param_dict["position"] = create_point(*param_dict["position"]) def __repr__(self): return ( diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py index 3c3a52485..0f3d9a22f 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py @@ -5,7 +5,7 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import Point3D +from artemis.utils import create_point def test_FGS_parameters_load_from_file(): @@ -18,8 +18,8 @@ def test_FGS_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - assert ispyb_params.position == Point3D(10, 20, 30) - assert ispyb_params.upper_left == Point3D(10, 20, 30) + assert ispyb_params.position == create_point(10, 20, 30) + assert ispyb_params.upper_left == create_point(10, 20, 30) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py index 6f42086fe..fce12c861 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py @@ -8,7 +8,7 @@ RotationInternalParameters, RotationScanParams, ) -from artemis.utils import Point3D +from artemis.utils import create_point def test_rotation_scan_param_validity(): @@ -53,8 +53,8 @@ def test_rotation_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - assert ispyb_params.position == Point3D(10, 20, 30) - assert ispyb_params.upper_left == Point3D(10, 20, 30) + assert ispyb_params.position == create_point(10, 20, 30) + assert ispyb_params.upper_left == create_point(10, 20, 30) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/parameters/tests/test_internal_parameters.py b/src/artemis/parameters/tests/test_internal_parameters.py index 7a2abddad..d3d7dc243 100644 --- a/src/artemis/parameters/tests/test_internal_parameters.py +++ b/src/artemis/parameters/tests/test_internal_parameters.py @@ -14,7 +14,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point TEST_PARAM_DICT = { "layer_1": { From da001545c67eee2f9bc7e00ec7824dc260c040b7 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Thu, 20 Apr 2023 11:33:02 +0100 Subject: [PATCH 09/22] fixed and added tests --- .../tests/test_fgs_internal_parameters.py | 5 +++-- .../test_rotation_internal_parameters.py | 5 +++-- src/artemis/unit_tests/test_utils.py | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 src/artemis/unit_tests/test_utils.py diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py index 0f3d9a22f..45b040b20 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py @@ -1,3 +1,4 @@ +import numpy as np from dodal.devices.det_dim_constants import EIGER2_X_16M_SIZE from dodal.devices.fast_grid_scan import GridScanParams @@ -18,8 +19,8 @@ def test_FGS_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - assert ispyb_params.position == create_point(10, 20, 30) - assert ispyb_params.upper_left == create_point(10, 20, 30) + np.testing.assert_array_equal(ispyb_params.position, create_point(10, 20, 30)) + np.testing.assert_array_equal(ispyb_params.upper_left, create_point(10, 20, 30)) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py index fce12c861..74a1bc359 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py @@ -1,5 +1,6 @@ from unittest.mock import MagicMock +import numpy as np from dodal.devices.det_dim_constants import EIGER2_X_16M_SIZE from dodal.devices.motors import XYZLimitBundle @@ -53,8 +54,8 @@ def test_rotation_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - assert ispyb_params.position == create_point(10, 20, 30) - assert ispyb_params.upper_left == create_point(10, 20, 30) + np.testing.assert_array_equal(ispyb_params.position, create_point(10, 20, 30)) + np.testing.assert_array_equal(ispyb_params.upper_left, create_point(10, 20, 30)) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/unit_tests/test_utils.py b/src/artemis/unit_tests/test_utils.py new file mode 100644 index 000000000..ad8181926 --- /dev/null +++ b/src/artemis/unit_tests/test_utils.py @@ -0,0 +1,22 @@ +import numpy as np +import pytest + +from artemis.utils import create_point + + +def test_create_point_on_invalid_number_of_args(): + with pytest.raises(TypeError): + create_point(1) + with pytest.raises(TypeError): + create_point() + with pytest.raises(TypeError): + create_point(7, 45, 23, 2, 1, 4) + + +def test_zero_array_created_on_none_type_args(): + np.testing.assert_equal(np.array([0, 0, 5]), create_point(None, None, 5)) + + +def test_create_point_turns_correct_array_size(): + assert create_point(5, 2).shape == (2,) + assert create_point(5, 2, 4).shape == (3,) From 8a6685942cbb5d4fcfbd2178aece1d35304285d0 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Thu, 20 Apr 2023 15:45:29 +0100 Subject: [PATCH 10/22] Fix other unit tests --- .../fgs/tests/test_zocalo_handler.py | 19 +++++++++---------- .../ispyb/ispyb_dataclass.py | 14 ++++++++++---- .../ispyb/store_in_ispyb.py | 12 ++++++------ .../unit_tests/test_store_in_ispyb.py | 6 +++--- src/artemis/unit_tests/test_utils.py | 4 ++-- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index 8d3c585ab..01499b02c 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -1,6 +1,7 @@ import operator from unittest.mock import MagicMock, call +import numpy as np import pytest from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( @@ -9,13 +10,11 @@ from artemis.external_interaction.callbacks.fgs.tests.conftest import TestData from artemis.external_interaction.exceptions import ISPyBDepositionNotMade from artemis.external_interaction.zocalo.zocalo_interaction import NoDiffractionFound -from artemis.utils import create_point +from artemis.parameters.external_parameters import from_file as default_raw_params from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) - -from artemis.parameters.external_parameters import from_file as default_raw_params - +from artemis.utils import create_point EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -112,7 +111,7 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal ) ) ) - assert found_centre == expected_centre_motor_coords + np.testing.assert_array_equal(found_centre, expected_centre_motor_coords) def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_THEN_fallback_centre_used( @@ -131,7 +130,7 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( 100 ) - assert found_centre == fallback_position + np.testing.assert_array_equal(found_centre, fallback_position) def test_GIVEN_ispyb_not_started_WHEN_trigger_zocalo_handler_THEN_raises_exception( @@ -179,13 +178,13 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b expected_centre_motor_coords = ( dummy_params.experiment_params.grid_position_to_motor_position( create_point( - expected_centre_grid_coords.x - 0.5, - expected_centre_grid_coords.y - 0.5, - expected_centre_grid_coords.z - 0.5, + expected_centre_grid_coords[0] - 0.5, + expected_centre_grid_coords[1] - 0.5, + expected_centre_grid_coords[2] - 0.5, ) ) ) - assert found_centre == expected_centre_motor_coords + np.testing.assert_array_equal(found_centre, expected_centre_motor_coords) expected_bbox_size = list(map(operator.sub, [8, 8, 7], [2, 2, 2])) assert found_bbox == expected_bbox_size diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 2116e86c5..9d2ac2eec 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -44,16 +44,16 @@ class IspybParams: upper_left: ndarray = field( # in px on the image metadata=config( - encoder=lambda mytuple: mytuple._asdict(), - decoder=lambda mydict: create_point(**mydict), + encoder=lambda my_array: str(my_array), + decoder=lambda my_list: ndarray(my_list), ) ) position: ndarray = field( # motor position metadata=config( - encoder=lambda mytuple: mytuple._asdict(), - decoder=lambda mydict: create_point(**mydict), + encoder=lambda my_array: str(my_array), + decoder=lambda my_list: ndarray(my_list), ) ) @@ -78,6 +78,12 @@ class IspybParams: slit_gap_size_x: Optional[float] = None slit_gap_size_y: Optional[float] = None + def __eq__(self, other) -> bool: + if not isinstance(other, IspybParams): + return NotImplemented + else: + return self.to_json() == other.to_json() + class Orientation(Enum): HORIZONTAL = "horizontal" diff --git a/src/artemis/external_interaction/ispyb/store_in_ispyb.py b/src/artemis/external_interaction/ispyb/store_in_ispyb.py index 9e7435d3e..218b97dec 100755 --- a/src/artemis/external_interaction/ispyb/store_in_ispyb.py +++ b/src/artemis/external_interaction/ispyb/store_in_ispyb.py @@ -84,8 +84,8 @@ def store_grid_scan(self, full_params: InternalParameters): self.omega_start = self.detector_params.omega_start self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_start self.upper_left = create_point( - self.ispyb_params.upper_left.x, - self.ispyb_params.upper_left.y, + self.ispyb_params.upper_left[0], + self.ispyb_params.upper_left[1], ) self.y_steps = full_params.experiment_params.y_steps self.y_step_size = full_params.experiment_params.y_step_size @@ -165,8 +165,8 @@ def _construct_comment(self) -> str: f"{self.y_steps} images in " f"{self.full_params.experiment_params.x_step_size*1e3} um by " f"{self.y_step_size*1e3} um steps. " - f"Top left (px): [{int(self.upper_left.x)},{int(self.upper_left.y)}], " - f"bottom right (px): [{bottom_right.x},{bottom_right.y}]." + f"Top left (px): [{int(self.upper_left[0])},{int(self.upper_left[1])}], " + f"bottom right (px): [{bottom_right[0]},{bottom_right[1]}]." ) @TRACER.start_as_current_span("store_ispyb_datacollection_table") @@ -314,8 +314,8 @@ def __prepare_second_scan_params(self): self.run_number += 1 self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_end self.upper_left = create_point( - self.ispyb_params.upper_left.x, - self.ispyb_params.upper_left.z, + self.ispyb_params.upper_left[0], + self.ispyb_params.upper_left[2], ) self.y_steps = self.full_params.experiment_params.z_steps self.y_step_size = self.full_params.experiment_params.z_step_size diff --git a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py index aaefa2f70..d74a5ed68 100644 --- a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py +++ b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py @@ -10,10 +10,10 @@ StoreInIspyb3D, ) from artemis.parameters.constants import SIM_ISPYB_CONFIG +from artemis.parameters.external_parameters import from_file as default_raw_params from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.parameters.external_parameters import from_file as default_raw_params from artemis.utils import create_point TEST_DATA_COLLECTION_IDS = [12, 13] @@ -155,8 +155,8 @@ def test_store_3d_grid_scan( assert dummy_ispyb_3d.y_step_size == dummy_params.experiment_params.z_step_size assert dummy_ispyb_3d.y_steps == dummy_params.experiment_params.z_steps - assert dummy_ispyb_3d.upper_left.x == x - assert dummy_ispyb_3d.upper_left.y == z + assert dummy_ispyb_3d.upper_left[0] == x + assert dummy_ispyb_3d.upper_left[1] == z def setup_mock_return_values(ispyb_conn): diff --git a/src/artemis/unit_tests/test_utils.py b/src/artemis/unit_tests/test_utils.py index ad8181926..018500996 100644 --- a/src/artemis/unit_tests/test_utils.py +++ b/src/artemis/unit_tests/test_utils.py @@ -13,10 +13,10 @@ def test_create_point_on_invalid_number_of_args(): create_point(7, 45, 23, 2, 1, 4) -def test_zero_array_created_on_none_type_args(): +def test_create_point_creates_zero_array_given_none_type_args(): np.testing.assert_equal(np.array([0, 0, 5]), create_point(None, None, 5)) -def test_create_point_turns_correct_array_size(): +def test_create_point_returns_correct_array_size(): assert create_point(5, 2).shape == (2,) assert create_point(5, 2, 4).shape == (3,) From 818a6e9cbd0e84858226fe5e00ec46b906eb3c91 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Fri, 21 Apr 2023 10:51:29 +0100 Subject: [PATCH 11/22] changed setup.cfg for dodal --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7a0cd45f9..e56b3f0fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = xarray doct databroker - dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@9f82cf522fddbdb415d36ead204ead380ab0648a + dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@27bc0c633ef975ce5d7ee03315f96c90a0e3dae6 [options.extras_require] dev = From 6a81f3998f37550c353905d9e119907ae986fb89 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 3 May 2023 16:41:53 +0100 Subject: [PATCH 12/22] replaced create_point() with np.array() --- setup.cfg | 2 +- .../experiment_plans/fast_grid_scan_plan.py | 15 +++++----- .../fgs/tests/test_fgs_callback_collection.py | 4 +-- .../fgs/tests/test_zocalo_handler.py | 29 +++++++++---------- .../callbacks/fgs/zocalo_callback.py | 13 +++++---- .../ispyb/ispyb_dataclass.py | 16 +++++----- .../ispyb/store_in_ispyb.py | 18 +++++++----- .../system_tests/conftest.py | 4 +-- .../system_tests/test_zocalo_system.py | 6 ++-- .../unit_tests/test_store_in_ispyb.py | 10 +++---- .../unit_tests/test_zocalo_interaction.py | 5 ++-- .../internal_parameters.py | 6 ++-- .../tests/test_fgs_internal_parameters.py | 5 ++-- .../test_rotation_internal_parameters.py | 5 ++-- .../unit_tests/test_fast_grid_scan_plan.py | 7 ++--- src/artemis/unit_tests/test_utils.py | 22 -------------- src/artemis/utils.py | 15 ---------- 17 files changed, 72 insertions(+), 110 deletions(-) delete mode 100644 src/artemis/unit_tests/test_utils.py delete mode 100644 src/artemis/utils.py diff --git a/setup.cfg b/setup.cfg index e56b3f0fb..c0215ee6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = xarray doct databroker - dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@27bc0c633ef975ce5d7ee03315f96c90a0e3dae6 + dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@f52611e1d730c2e57b8b2e3ead82064167f824f6 [options.extras_require] dev = diff --git a/src/artemis/experiment_plans/fast_grid_scan_plan.py b/src/artemis/experiment_plans/fast_grid_scan_plan.py index 80f5201fb..4a7e80492 100755 --- a/src/artemis/experiment_plans/fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/fast_grid_scan_plan.py @@ -5,6 +5,7 @@ import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp +import numpy as np from bluesky import RunEngine from bluesky.utils import ProgressBarManager from dodal import i03 @@ -22,7 +23,6 @@ Undulator, Zebra, ) -from numpy import ndarray import artemis.log from artemis.device_setup_plans.setup_zebra_for_fgs import ( @@ -41,7 +41,6 @@ SIM_BEAMLINE, ) from artemis.tracing import TRACER -from artemis.utils import create_point if TYPE_CHECKING: from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( @@ -155,7 +154,7 @@ def read_hardware_for_ispyb( @bpp.run_decorator(md={"subplan_name": "move_xyz"}) def move_xyz( sample_motors, - xray_centre_motor_position: ndarray, + xray_centre_motor_position: np.ndarray, md={ "plan_name": "move_xyz", }, @@ -251,10 +250,12 @@ def run_gridscan_and_move( and moves to the centre of mass determined by zocalo""" # We get the initial motor positions so we can return to them on zocalo failure - initial_xyz = create_point( - (yield from bps.rd(fgs_composite.sample_motors.x)), - (yield from bps.rd(fgs_composite.sample_motors.y)), - (yield from bps.rd(fgs_composite.sample_motors.z)), + initial_xyz = np.array( + [ + (yield from bps.rd(fgs_composite.sample_motors.x)), + (yield from bps.rd(fgs_composite.sample_motors.y)), + (yield from bps.rd(fgs_composite.sample_motors.z)), + ] ) yield from setup_zebra_for_fgs(fgs_composite.zebra) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py index 2f9d6d7eb..74d18a390 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_fgs_callback_collection.py @@ -1,5 +1,6 @@ from unittest.mock import MagicMock +import numpy as np import pytest from bluesky.run_engine import RunEngine from dodal.devices.eiger import DetectorParams, EigerDetector @@ -16,7 +17,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point def test_callback_collection_init(): @@ -92,7 +92,7 @@ def test_communicator_in_composite_run( callbacks.zocalo_handler._wait_for_result = MagicMock() callbacks.zocalo_handler._run_end = MagicMock() callbacks.zocalo_handler._run_start = MagicMock() - callbacks.zocalo_handler.xray_centre_motor_position = create_point(1, 2, 3) + callbacks.zocalo_handler.xray_centre_motor_position = np.array([1, 2, 3]) fast_grid_scan_composite = FGSComposite() # this is where it's currently getting stuck: diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index 01499b02c..fb2da510d 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -14,7 +14,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -85,7 +84,7 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) - expected_centre_grid_coords = create_point(1, 2, 3) + expected_centre_grid_coords = np.array([1, 2, 3]) single_crystal_result = [ { "max_voxel": [1, 2, 3], @@ -98,17 +97,13 @@ def test_zocalo_called_to_wait_on_results_when_communicator_wait_for_results_cal single_crystal_result ) - found_centre = callbacks.zocalo_handler.wait_for_results(create_point(0, 0, 0))[0] + found_centre = callbacks.zocalo_handler.wait_for_results(np.array([0, 0, 0]))[0] callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( 100 ) expected_centre_motor_coords = ( dummy_params.experiment_params.grid_position_to_motor_position( - create_point( - expected_centre_grid_coords[0] - 0.5, - expected_centre_grid_coords[1] - 0.5, - expected_centre_grid_coords[2] - 0.5, - ) + expected_centre_grid_coords - 0.5 ) ) np.testing.assert_array_equal(found_centre, expected_centre_motor_coords) @@ -124,7 +119,7 @@ def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_ NoDiffractionFound() ) - fallback_position = create_point(1, 2, 3) + fallback_position = np.array([1, 2, 3]) found_centre = callbacks.zocalo_handler.wait_for_results(fallback_position)[0] callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( @@ -149,11 +144,11 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b callbacks = FGSCallbackCollection.from_params(dummy_params) mock_zocalo_functions(callbacks) callbacks.ispyb_handler.ispyb_ids = (0, 0, 100) - expected_centre_grid_coords = create_point(4, 6, 2) + expected_centre_grid_coords = np.array([4, 6, 2]) multi_crystal_result = [ { "max_voxel": [1, 2, 3], - "centre_of_mass": create_point(3, 11, 11), + "centre_of_mass": np.array([3, 11, 11]), "bounding_box": [[1, 1, 1], [3, 3, 3]], "n_voxels": 2, "total_count": 192512.0, @@ -170,17 +165,19 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b multi_crystal_result ) found_centre, found_bbox = callbacks.zocalo_handler.wait_for_results( - create_point(0, 0, 0) + np.array([0, 0, 0]) ) callbacks.zocalo_handler.zocalo_interactor.wait_for_result.assert_called_once_with( 100 ) expected_centre_motor_coords = ( dummy_params.experiment_params.grid_position_to_motor_position( - create_point( - expected_centre_grid_coords[0] - 0.5, - expected_centre_grid_coords[1] - 0.5, - expected_centre_grid_coords[2] - 0.5, + np.array( + [ + expected_centre_grid_coords[0] - 0.5, + expected_centre_grid_coords[1] - 0.5, + expected_centre_grid_coords[2] - 0.5, + ] ) ) ) diff --git a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py index 85f253a3b..8f6154244 100644 --- a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py @@ -4,6 +4,7 @@ import time from typing import Callable, Optional +import numpy as np from bluesky.callbacks import CallbackBase from numpy import ndarray @@ -19,7 +20,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point class FGSZocaloCallback(CallbackBase): @@ -111,7 +111,10 @@ def wait_for_results(self, fallback_xyz: ndarray) -> tuple[ndarray, Optional[lis ) ) ) - nicely_formatted_com = [f"{com:.2f}" for com in res["centre_of_mass"]] + + nicely_formatted_com = [ + f"{np.round(com,2)}" for com in res["centre_of_mass"] + ] crystal_summary += ( f"Crystal {n+1}: " f"Strength {res['total_count']}; " @@ -120,11 +123,11 @@ def wait_for_results(self, fallback_xyz: ndarray) -> tuple[ndarray, Optional[lis ) self.ispyb.append_to_comment(crystal_summary) - raw_centre = create_point(*(raw_results[0]["centre_of_mass"])) + raw_centre = np.array([*(raw_results[0]["centre_of_mass"])]) # _wait_for_result returns the centre of the grid box, but we want the corner - results = create_point( - raw_centre[0] - 0.5, raw_centre[1] - 0.5, raw_centre[2] - 0.5 + results = np.array( + [raw_centre[0] - 0.5, raw_centre[1] - 0.5, raw_centre[2] - 0.5] ) xray_centre = self.grid_position_to_motor_position(results) diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 9d2ac2eec..5160b3640 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -2,10 +2,8 @@ from enum import Enum from typing import List, Optional +import numpy as np from dataclasses_json import config, dataclass_json -from numpy import ndarray - -from artemis.utils import create_point ISPYB_PARAM_DEFAULTS = { "sample_id": None, @@ -14,8 +12,8 @@ "microns_per_pixel_x": 0.0, "microns_per_pixel_y": 0.0, # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels - "upper_left": create_point(0, 0, 0), - "position": create_point(0, 0, 0), + "upper_left": np.array([0, 0, 0]), + "position": np.array([0, 0, 0]), "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], "transmission": 1.0, @@ -41,19 +39,19 @@ class IspybParams: microns_per_pixel_x: float microns_per_pixel_y: float - upper_left: ndarray = field( + upper_left: np.ndarray = field( # in px on the image metadata=config( encoder=lambda my_array: str(my_array), - decoder=lambda my_list: ndarray(my_list), + decoder=lambda my_list: np.ndarray(my_list), ) ) - position: ndarray = field( + position: np.ndarray = field( # motor position metadata=config( encoder=lambda my_array: str(my_array), - decoder=lambda my_list: ndarray(my_list), + decoder=lambda my_list: np.ndarray(my_list), ) ) diff --git a/src/artemis/external_interaction/ispyb/store_in_ispyb.py b/src/artemis/external_interaction/ispyb/store_in_ispyb.py index 218b97dec..feef623db 100755 --- a/src/artemis/external_interaction/ispyb/store_in_ispyb.py +++ b/src/artemis/external_interaction/ispyb/store_in_ispyb.py @@ -8,12 +8,12 @@ import dodal.devices.oav.utils as oav_utils import ispyb import ispyb.sqlalchemy +import numpy as np from sqlalchemy.connectors import Connector from artemis.external_interaction.ispyb.ispyb_dataclass import Orientation from artemis.log import LOGGER from artemis.tracing import TRACER -from artemis.utils import create_point if TYPE_CHECKING: from artemis.parameters.internal_parameters import InternalParameters @@ -83,9 +83,11 @@ def store_grid_scan(self, full_params: InternalParameters): self.run_number = self.detector_params.run_number self.omega_start = self.detector_params.omega_start self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_start - self.upper_left = create_point( - self.ispyb_params.upper_left[0], - self.ispyb_params.upper_left[1], + self.upper_left = np.array( + [ + self.ispyb_params.upper_left[0], + self.ispyb_params.upper_left[1], + ] ) self.y_steps = full_params.experiment_params.y_steps self.y_step_size = full_params.experiment_params.y_step_size @@ -313,9 +315,11 @@ def __prepare_second_scan_params(self): self.omega_start += 90 self.run_number += 1 self.xtal_snapshots = self.ispyb_params.xtal_snapshots_omega_end - self.upper_left = create_point( - self.ispyb_params.upper_left[0], - self.ispyb_params.upper_left[2], + self.upper_left = np.array( + [ + self.ispyb_params.upper_left[0], + self.ispyb_params.upper_left[2], + ] ) self.y_steps = self.full_params.experiment_params.z_steps self.y_step_size = self.full_params.experiment_params.z_step_size diff --git a/src/artemis/external_interaction/system_tests/conftest.py b/src/artemis/external_interaction/system_tests/conftest.py index 036f02800..d02ec9cc6 100644 --- a/src/artemis/external_interaction/system_tests/conftest.py +++ b/src/artemis/external_interaction/system_tests/conftest.py @@ -3,6 +3,7 @@ from typing import Callable import ispyb.sqlalchemy +import numpy as np import pytest from ispyb.sqlalchemy import DataCollection from sqlalchemy import create_engine @@ -17,7 +18,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point ISPYB_CONFIG = "/dls_sw/dasc/mariadb/credentials/ispyb-dev.cfg" @@ -79,7 +79,7 @@ def fetch_comment() -> Callable: @pytest.fixture def dummy_params(): dummy_params = FGSInternalParameters(default_raw_params()) - dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) + dummy_params.artemis_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 dummy_params.artemis_params.ispyb_params.visit_path = ( diff --git a/src/artemis/external_interaction/system_tests/test_zocalo_system.py b/src/artemis/external_interaction/system_tests/test_zocalo_system.py index 3a636db14..d74f2158b 100644 --- a/src/artemis/external_interaction/system_tests/test_zocalo_system.py +++ b/src/artemis/external_interaction/system_tests/test_zocalo_system.py @@ -1,3 +1,4 @@ +import numpy as np import pytest from artemis.external_interaction.callbacks.fgs.fgs_callback_collection import ( @@ -12,7 +13,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point @pytest.mark.s03 @@ -31,7 +31,7 @@ def test_when_running_start_stop_then_get_expected_returned_results(zocalo_env): @pytest.fixture def run_zocalo_with_dev_ispyb(dummy_params: FGSInternalParameters, dummy_ispyb_3d): - def inner(sample_name="", fallback=create_point(0, 0, 0)): + def inner(sample_name="", fallback=np.array([0, 0, 0])): dummy_params.artemis_params.detector_params.prefix = sample_name zc: FGSZocaloCallback = FGSCallbackCollection.from_params( dummy_params @@ -51,7 +51,7 @@ def inner(sample_name="", fallback=create_point(0, 0, 0)): def test_given_a_result_with_no_diffraction_when_zocalo_called_then_move_to_fallback( run_zocalo_with_dev_ispyb, zocalo_env ): - fallback = create_point(1, 2, 3) + fallback = np.array([1, 2, 3]) zc, centre = run_zocalo_with_dev_ispyb("NO_DIFF", fallback) assert centre == fallback diff --git a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py index d74a5ed68..a52d1020f 100644 --- a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py +++ b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py @@ -1,6 +1,7 @@ import re from unittest.mock import MagicMock, Mock, mock_open, patch +import numpy as np import pytest from ispyb.sp.mxacquisition import MXAcquisition from mockito import mock, when @@ -14,7 +15,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point TEST_DATA_COLLECTION_IDS = [12, 13] TEST_DATA_COLLECTION_GROUP_ID = 34 @@ -28,7 +28,7 @@ @pytest.fixture def dummy_params(): dummy_params = FGSInternalParameters(default_raw_params()) - dummy_params.artemis_params.ispyb_params.upper_left = create_point(100, 100, 50) + dummy_params.artemis_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 return dummy_params @@ -129,7 +129,7 @@ def test_store_3d_grid_scan( y = 1 z = 2 - dummy_params.artemis_params.ispyb_params.upper_left = create_point(x, y, z) + dummy_params.artemis_params.ispyb_params.upper_left = np.array([x, y, z]) dummy_params.experiment_params.z_step_size = 0.2 assert dummy_ispyb_3d.experiment_type == "Mesh3D" @@ -315,8 +315,8 @@ def test_ispyb_deposition_rounds_to_int( mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition ) mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection - dummy_ispyb.full_params.artemis_params.ispyb_params.upper_left = create_point( - 0.01, 100, 50 + dummy_ispyb.full_params.artemis_params.ispyb_params.upper_left = np.array( + [0.01, 100, 50] ) dummy_ispyb.begin_deposition() mock_upsert_call_args = mock_upsert_data_collection.call_args_list[0][0] diff --git a/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py b/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py index ee76be307..97ed315ae 100644 --- a/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py +++ b/src/artemis/external_interaction/unit_tests/test_zocalo_interaction.py @@ -16,7 +16,6 @@ ZocaloInteractor, ) from artemis.parameters.constants import SIM_ZOCALO_ENV -from artemis.utils import create_point EXPECTED_DCID = 100 EXPECTED_RUN_START_MESSAGE = {"event": "start", "ispyb_dcid": EXPECTED_DCID} @@ -138,9 +137,9 @@ def test_when_message_recieved_from_zocalo_then_point_returned( return_value = future.result() assert type(return_value) == list - returned_com = create_point(*return_value[0]["centre_of_mass"]) + returned_com = np.array([*return_value[0]["centre_of_mass"]]) np.testing.assert_array_almost_equal( - returned_com, create_point(*centre_of_mass_coords) + returned_com, np.array([*centre_of_mass_coords]) ) diff --git a/src/artemis/parameters/internal_parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters/internal_parameters.py index e20da9a85..e75f6a25b 100644 --- a/src/artemis/parameters/internal_parameters/internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/internal_parameters.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict +import numpy as np from dodal.devices.eiger import DetectorParams from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase @@ -16,7 +17,6 @@ SIM_INSERTION_PREFIX, SIM_ZOCALO_ENV, ) -from artemis.utils import create_point class ArtemisParameters: @@ -181,8 +181,8 @@ def artemis_param_preprocessing(self, param_dict: dict[str, Any]): """ param_dict["num_images"] = self.experiment_params.get_num_images() - param_dict["upper_left"] = create_point(*param_dict["upper_left"]) - param_dict["position"] = create_point(*param_dict["position"]) + param_dict["upper_left"] = np.array([*param_dict["upper_left"]]) + param_dict["position"] = np.array([*param_dict["position"]]) def __repr__(self): return ( diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py index 45b040b20..fba748161 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_fgs_internal_parameters.py @@ -6,7 +6,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point def test_FGS_parameters_load_from_file(): @@ -19,8 +18,8 @@ def test_FGS_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - np.testing.assert_array_equal(ispyb_params.position, create_point(10, 20, 30)) - np.testing.assert_array_equal(ispyb_params.upper_left, create_point(10, 20, 30)) + np.testing.assert_array_equal(ispyb_params.position, np.array([10, 20, 30])) + np.testing.assert_array_equal(ispyb_params.upper_left, np.array([10, 20, 30])) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py index 74a1bc359..ebd905850 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/tests/test_rotation_internal_parameters.py @@ -9,7 +9,6 @@ RotationInternalParameters, RotationScanParams, ) -from artemis.utils import create_point def test_rotation_scan_param_validity(): @@ -54,8 +53,8 @@ def test_rotation_parameters_load_from_file(): ispyb_params = internal_parameters.artemis_params.ispyb_params - np.testing.assert_array_equal(ispyb_params.position, create_point(10, 20, 30)) - np.testing.assert_array_equal(ispyb_params.upper_left, create_point(10, 20, 30)) + np.testing.assert_array_equal(ispyb_params.position, np.array([10, 20, 30])) + np.testing.assert_array_equal(ispyb_params.upper_left, np.array([10, 20, 30])) detector_params = internal_parameters.artemis_params.detector_params diff --git a/src/artemis/unit_tests/test_fast_grid_scan_plan.py b/src/artemis/unit_tests/test_fast_grid_scan_plan.py index be4ad23a4..071d33f3d 100644 --- a/src/artemis/unit_tests/test_fast_grid_scan_plan.py +++ b/src/artemis/unit_tests/test_fast_grid_scan_plan.py @@ -46,7 +46,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( FGSInternalParameters, ) -from artemis.utils import create_point @pytest.fixture @@ -239,7 +238,7 @@ def test_results_passed_to_move_motors( set_up_logging_handlers(logging_level="INFO", dev_mode=True) RE.subscribe(VerbosePlanExecutionLoggingCallback()) motor_position = test_params.experiment_params.grid_position_to_motor_position( - create_point(1, 2, 3) + np.array([1, 2, 3]) ) RE(move_xyz(fake_fgs_composite.sample_motors, motor_position)) bps_mv.assert_called_once_with( @@ -279,7 +278,7 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] np.testing.assert_array_equal( - array_arg, create_point(0.05, 0.15000000000000002, 0.25) + array_arg, np.array([0.05, 0.15000000000000002, 0.25], dtype=np.float16) ) move_xyz.assert_called_once() @@ -316,7 +315,7 @@ def test_logging_within_plan( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] np.testing.assert_array_almost_equal( - array_arg, create_point(0.05, 0.15000000000000002, 0.25) + array_arg, np.array([0.05, 0.15000000000000002, 0.25], dtype=np.float16) ) move_xyz.assert_called_once() diff --git a/src/artemis/unit_tests/test_utils.py b/src/artemis/unit_tests/test_utils.py deleted file mode 100644 index 018500996..000000000 --- a/src/artemis/unit_tests/test_utils.py +++ /dev/null @@ -1,22 +0,0 @@ -import numpy as np -import pytest - -from artemis.utils import create_point - - -def test_create_point_on_invalid_number_of_args(): - with pytest.raises(TypeError): - create_point(1) - with pytest.raises(TypeError): - create_point() - with pytest.raises(TypeError): - create_point(7, 45, 23, 2, 1, 4) - - -def test_create_point_creates_zero_array_given_none_type_args(): - np.testing.assert_equal(np.array([0, 0, 5]), create_point(None, None, 5)) - - -def test_create_point_returns_correct_array_size(): - assert create_point(5, 2).shape == (2,) - assert create_point(5, 2, 4).shape == (3,) diff --git a/src/artemis/utils.py b/src/artemis/utils.py deleted file mode 100644 index 9deacfc01..000000000 --- a/src/artemis/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import numpy as np - - -def create_point(*args): - args = list(args) - for index, arg in enumerate(args): - if args[index] is None: - args[index] = 0 - - if len(args) == 2: - return np.array([args[0], args[1]], dtype=np.float16) - elif len(args) == 3: - return np.array([args[0], args[1], args[2]], dtype=np.float16) - else: - raise TypeError("Invalid number of arguments") From 4125cd82aa4fb2b51bc3a349906e4bf1e9f0622b Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 3 May 2023 16:48:34 +0100 Subject: [PATCH 13/22] fix unit tests --- src/artemis/unit_tests/test_fast_grid_scan_plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/artemis/unit_tests/test_fast_grid_scan_plan.py b/src/artemis/unit_tests/test_fast_grid_scan_plan.py index 071d33f3d..0a4845405 100644 --- a/src/artemis/unit_tests/test_fast_grid_scan_plan.py +++ b/src/artemis/unit_tests/test_fast_grid_scan_plan.py @@ -278,7 +278,7 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] np.testing.assert_array_equal( - array_arg, np.array([0.05, 0.15000000000000002, 0.25], dtype=np.float16) + array_arg, np.array([0.05, 0.15000000000000002, 0.25]) ) move_xyz.assert_called_once() @@ -315,7 +315,7 @@ def test_logging_within_plan( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] np.testing.assert_array_almost_equal( - array_arg, np.array([0.05, 0.15000000000000002, 0.25], dtype=np.float16) + array_arg, np.array([0.05, 0.15000000000000002, 0.25]) ) move_xyz.assert_called_once() From a6a4a3cc61ab107f0c95aeb3f89c7703657d865a Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 3 May 2023 16:57:55 +0100 Subject: [PATCH 14/22] minor numpifying --- .../callbacks/fgs/tests/test_zocalo_handler.py | 4 ++-- .../external_interaction/callbacks/fgs/zocalo_callback.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index fb2da510d..91fc49b6e 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -183,5 +183,5 @@ def test_multiple_results_from_zocalo_sorted_by_total_count_returns_centre_and_b ) np.testing.assert_array_equal(found_centre, expected_centre_motor_coords) - expected_bbox_size = list(map(operator.sub, [8, 8, 7], [2, 2, 2])) - assert found_bbox == expected_bbox_size + expected_bbox_size = np.array([8, 8, 7]) - np.array([2, 2, 2]) + np.testing.assert_array_equal(found_bbox, expected_bbox_size) diff --git a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py index 8f6154244..aaa2488a3 100644 --- a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py @@ -105,11 +105,7 @@ def wait_for_results(self, fallback_xyz: ndarray) -> tuple[ndarray, Optional[lis bboxes = [] for n, res in enumerate(raw_results): bboxes.append( - list( - map( - operator.sub, res["bounding_box"][1], res["bounding_box"][0] - ) - ) + np.array(res["bounding_box"][1]) - np.array(res["bounding_box"][0]) ) nicely_formatted_com = [ From a9457ce044e7bb48d33d4bc31da5f00dd0b09f15 Mon Sep 17 00:00:00 2001 From: Ollie Silvester Date: Wed, 3 May 2023 16:59:12 +0100 Subject: [PATCH 15/22] remove unused imports --- .../callbacks/fgs/tests/test_zocalo_handler.py | 1 - .../external_interaction/callbacks/fgs/zocalo_callback.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py index 91fc49b6e..121ddaec0 100644 --- a/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py +++ b/src/artemis/external_interaction/callbacks/fgs/tests/test_zocalo_handler.py @@ -1,4 +1,3 @@ -import operator from unittest.mock import MagicMock, call import numpy as np diff --git a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py index aaa2488a3..93d9798ab 100644 --- a/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py +++ b/src/artemis/external_interaction/callbacks/fgs/zocalo_callback.py @@ -1,6 +1,5 @@ from __future__ import annotations -import operator import time from typing import Callable, Optional From 809785b63c56fc36e62c001baa361caec2ff758e Mon Sep 17 00:00:00 2001 From: Oliver Silvester Date: Mon, 15 May 2023 14:00:06 +0100 Subject: [PATCH 16/22] changed setup to correct dodal --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5c4b8feaf..574801e69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = xarray doct databroker - dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@0.0.3 + dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@f6ca7ceee77a8bbda2ab0f3598df473b4f9cf2c6 [options.extras_require] dev = From d3c744248355607053bea7bdf656c7112dd00788 Mon Sep 17 00:00:00 2001 From: David Perl Date: Wed, 31 May 2023 14:23:05 +0100 Subject: [PATCH 17/22] update merged changes --- src/artemis/experiment_plans/full_grid_scan.py | 3 +-- .../experiment_plans/oav_grid_detection_plan.py | 10 +++++----- .../experiment_plans/tests/test_fast_grid_scan_plan.py | 1 + .../external_interaction/ispyb/ispyb_dataclass.py | 7 +++++++ .../plan_specific/fgs_internal_params.py | 4 ++-- .../plan_specific/rotation_scan_internal_params.py | 4 ++-- 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/artemis/experiment_plans/full_grid_scan.py b/src/artemis/experiment_plans/full_grid_scan.py index 5ad372400..c6f3bfdb5 100644 --- a/src/artemis/experiment_plans/full_grid_scan.py +++ b/src/artemis/experiment_plans/full_grid_scan.py @@ -25,7 +25,6 @@ from artemis.parameters.internal_parameters.plan_specific.fgs_internal_params import ( GridScanParams, ) -from artemis.utils.utils import Point3D if TYPE_CHECKING: from artemis.parameters.internal_parameters.plan_specific.grid_scan_with_edge_detect_params import ( @@ -104,7 +103,7 @@ def detect_grid_and_do_gridscan(): parameters.artemis_params.ispyb_params.xtal_snapshots_omega_end = ( out_snapshot_filenames[1] ) - parameters.artemis_params.ispyb_params.upper_left = Point3D(**out_upper_left) + parameters.artemis_params.ispyb_params.upper_left = out_upper_left fgs_params.__post_init__() diff --git a/src/artemis/experiment_plans/oav_grid_detection_plan.py b/src/artemis/experiment_plans/oav_grid_detection_plan.py index 3d7a65156..7af0d0142 100644 --- a/src/artemis/experiment_plans/oav_grid_detection_plan.py +++ b/src/artemis/experiment_plans/oav_grid_detection_plan.py @@ -32,7 +32,7 @@ def grid_detection_plan( snapshot_template: str, snapshot_dir: str, out_snapshot_filenames: List[List[str]], - out_upper_left: Dict, + out_upper_left: list[float] | np.ndarray, width=600, box_size_microns=20, ): @@ -57,7 +57,7 @@ def grid_detection_main_plan( snapshot_template: str, snapshot_dir: str, out_snapshot_filenames: List[List[str]], - out_upper_left: Dict, + out_upper_left: list[float] | np.ndarray, grid_width_px: int, box_size_um: float, ): @@ -133,10 +133,10 @@ def grid_detection_main_plan( upper_left = (tip_x_px, min_y) if angle == 0: - out_upper_left["x"] = int(tip_x_px) - out_upper_left["y"] = int(min_y) + out_upper_left[0] = int(tip_x_px) + out_upper_left[1] = int(min_y) else: - out_upper_left["z"] = int(min_y) + out_upper_left[2] = int(min_y) yield from bps.abs_set(oav.snapshot.top_left_x, upper_left[0]) yield from bps.abs_set(oav.snapshot.top_left_y, upper_left[1]) diff --git a/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py b/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py index 97e5c362f..6c6d43f74 100644 --- a/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py @@ -5,6 +5,7 @@ import numpy as np import pytest from bluesky.run_engine import RunEngine +from dodal.devices.aperturescatterguard import AperturePositions from dodal.devices.det_dim_constants import ( EIGER2_X_4M_DIMENSION, EIGER_TYPE_EIGER2_X_4M, diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 63fe9bb07..6717945de 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -35,11 +35,17 @@ class IspybParams(BaseModel): visit_path: str microns_per_pixel_x: float microns_per_pixel_y: float + upper_left: np.ndarray + position: np.ndarray + + class Config: + arbitrary_types_allowed = True @validator("upper_left", pre=True) def _parse_upper_left( cls, upper_left: list[int | float] | np.ndarray, values: dict[str, Any] ) -> np.ndarray: + assert len(upper_left) == 3 if isinstance(upper_left, np.ndarray): return upper_left return np.array(upper_left) @@ -48,6 +54,7 @@ def _parse_upper_left( def _parse_position( cls, position: list[int | float] | np.ndarray, values: dict[str, Any] ) -> np.ndarray: + assert len(position) == 3 if isinstance(position, np.ndarray): return position return np.array(position) diff --git a/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py b/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py index f80219618..83d88dc33 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/fgs_internal_params.py @@ -2,11 +2,11 @@ from typing import Any +import numpy as np from dodal.devices.detector import TriggerMode from dodal.devices.fast_grid_scan import GridScanParams from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils.utils import Point3D class FGSInternalParameters(InternalParameters): @@ -19,4 +19,4 @@ def artemis_param_preprocessing(self, param_dict: dict[str, Any]): param_dict["num_triggers"] = param_dict["num_images"] param_dict["num_images_per_trigger"] = 1 param_dict["trigger_mode"] = TriggerMode.FREE_RUN - param_dict["upper_left"] = Point3D(*param_dict["upper_left"]) + param_dict["upper_left"] = np.array(param_dict["upper_left"]) diff --git a/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py b/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py index 54cf79b2f..a9a9a128c 100644 --- a/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py +++ b/src/artemis/parameters/internal_parameters/plan_specific/rotation_scan_internal_params.py @@ -3,12 +3,12 @@ from dataclasses import dataclass from typing import Any, Optional +import numpy as np from dataclasses_json import DataClassJsonMixin from dodal.devices.motors import XYZLimitBundle from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase from artemis.parameters.internal_parameters import InternalParameters -from artemis.utils.utils import Point3D @dataclass @@ -66,4 +66,4 @@ def artemis_param_preprocessing(self, param_dict: dict[str, Any]): param_dict["omega_increment"] = 0 param_dict["num_triggers"] = 1 param_dict["num_images_per_trigger"] = param_dict["num_images"] - param_dict["upper_left"] = Point3D(*param_dict["upper_left"]) + param_dict["upper_left"] = np.array(param_dict["upper_left"]) From d82303daf3f16087df2cd1ec2d375a234b37fc03 Mon Sep 17 00:00:00 2001 From: David Perl Date: Wed, 31 May 2023 14:40:23 +0100 Subject: [PATCH 18/22] fix encoding --- .../experiment_plans/tests/test_fast_grid_scan_plan.py | 8 ++------ src/artemis/external_interaction/ispyb/ispyb_dataclass.py | 1 + .../parameters/internal_parameters/internal_parameters.py | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py b/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py index 6c6d43f74..7577aa48a 100644 --- a/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py +++ b/src/artemis/experiment_plans/tests/test_fast_grid_scan_plan.py @@ -275,9 +275,7 @@ def test_individual_plans_triggered_once_and_only_once_in_composite_run( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] - np.testing.assert_array_equal( - array_arg, np.array([0.05, 0.15000000000000002, 0.25]) - ) + np.testing.assert_allclose(array_arg, np.array([-0.05, 0.05, 0.15])) move_xyz.assert_called_once() @@ -312,9 +310,7 @@ def test_logging_within_plan( run_gridscan.assert_called_once_with(fake_fgs_composite, test_params) array_arg = move_xyz.call_args.args[1] - np.testing.assert_array_almost_equal( - array_arg, np.array([0.05, 0.15000000000000002, 0.25]) - ) + np.testing.assert_array_almost_equal(array_arg, np.array([-0.05, 0.05, 0.15])) move_xyz.assert_called_once() diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index 6717945de..c4a6f236f 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -40,6 +40,7 @@ class IspybParams(BaseModel): class Config: arbitrary_types_allowed = True + json_encoders = {np.ndarray: lambda a: a.tolist()} @validator("upper_left", pre=True) def _parse_upper_left( diff --git a/src/artemis/parameters/internal_parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters/internal_parameters.py index 3875ba783..c323c7f8a 100644 --- a/src/artemis/parameters/internal_parameters/internal_parameters.py +++ b/src/artemis/parameters/internal_parameters/internal_parameters.py @@ -4,6 +4,7 @@ import numpy as np from dodal.devices.eiger import DetectorParams from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase +from numpy import ndarray from pydantic import BaseModel import artemis.parameters.external_parameters as raw_parameters @@ -28,6 +29,9 @@ class ArtemisParameters(BaseModel): detector_params: DetectorParams = DetectorParams(**DETECTOR_PARAM_DEFAULTS) ispyb_params: IspybParams = IspybParams(**ISPYB_PARAM_DEFAULTS) + class Config: + json_encoders = {ndarray: lambda a: a.tolist()} + def __repr__(self): return ( "artemis_params:\n" From 86cb2a477d0030a58f566f0d7d5d155428bda798 Mon Sep 17 00:00:00 2001 From: David Perl Date: Wed, 31 May 2023 14:40:58 +0100 Subject: [PATCH 19/22] remove unused import --- src/artemis/experiment_plans/oav_grid_detection_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/artemis/experiment_plans/oav_grid_detection_plan.py b/src/artemis/experiment_plans/oav_grid_detection_plan.py index 7af0d0142..e54456b89 100644 --- a/src/artemis/experiment_plans/oav_grid_detection_plan.py +++ b/src/artemis/experiment_plans/oav_grid_detection_plan.py @@ -2,7 +2,7 @@ import math from os.path import join as path_join -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, List import bluesky.plan_stubs as bps import numpy as np From 4e96892fbb18147a19b9c6c701c5039f3c853c65 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Jun 2023 10:01:30 +0100 Subject: [PATCH 20/22] fix merge remnants --- src/artemis/parameters/internal_parameters.py | 1 - src/artemis/parameters/plan_specific/fgs_internal_params.py | 6 +++--- .../plan_specific/grid_scan_with_edge_detect_params.py | 6 +++--- .../plan_specific/rotation_scan_internal_params.py | 6 +++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/artemis/parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters.py index c7d1c0d11..527052f08 100644 --- a/src/artemis/parameters/internal_parameters.py +++ b/src/artemis/parameters/internal_parameters.py @@ -2,7 +2,6 @@ from typing import Any from dodal.devices.eiger import DetectorParams -from numpy import ndarray from pydantic import BaseModel, root_validator from semver import Version diff --git a/src/artemis/parameters/plan_specific/fgs_internal_params.py b/src/artemis/parameters/plan_specific/fgs_internal_params.py index 2c3094061..5935bd2a1 100644 --- a/src/artemis/parameters/plan_specific/fgs_internal_params.py +++ b/src/artemis/parameters/plan_specific/fgs_internal_params.py @@ -2,6 +2,7 @@ from typing import Any +import numpy as np from dodal.devices.detector import TriggerMode from dodal.devices.fast_grid_scan import GridScanParams from pydantic import validator @@ -12,7 +13,6 @@ extract_artemis_params_from_flat_dict, extract_experiment_params_from_flat_dict, ) -from artemis.utils.utils import Point3D class FGSInternalParameters(InternalParameters): @@ -43,11 +43,11 @@ def _preprocess_artemis_params( ): experiment_params: GridScanParams = values["experiment_params"] all_params["num_images"] = experiment_params.get_num_images() - all_params["position"] = Point3D(*all_params["position"]) + all_params["position"] = np.array(all_params["position"]) all_params["omega_increment"] = 0 all_params["num_triggers"] = all_params["num_images"] all_params["num_images_per_trigger"] = 1 all_params["trigger_mode"] = TriggerMode.FREE_RUN - all_params["upper_left"] = Point3D(*all_params["upper_left"]) + all_params["upper_left"] = np.array(all_params["upper_left"]) artemis_param_dict = extract_artemis_params_from_flat_dict(all_params) return ArtemisParameters(**artemis_param_dict) diff --git a/src/artemis/parameters/plan_specific/grid_scan_with_edge_detect_params.py b/src/artemis/parameters/plan_specific/grid_scan_with_edge_detect_params.py index dd9c12297..ab2bc7d7f 100644 --- a/src/artemis/parameters/plan_specific/grid_scan_with_edge_detect_params.py +++ b/src/artemis/parameters/plan_specific/grid_scan_with_edge_detect_params.py @@ -2,6 +2,7 @@ from typing import Any +import numpy as np from dataclasses_json import DataClassJsonMixin from dodal.devices.detector import TriggerMode from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase @@ -16,7 +17,6 @@ flatten_dict, get_extracted_experiment_and_flat_artemis_params, ) -from artemis.utils.utils import Point3D @dataclass @@ -61,10 +61,10 @@ def _preprocess_artemis_params( ): experiment_params: GridScanWithEdgeDetectParams = values["experiment_params"] all_params["num_images"] = experiment_params.get_num_images() - all_params["position"] = Point3D(*all_params["position"]) + all_params["position"] = np.array(all_params["position"]) all_params["omega_increment"] = 0 all_params["num_triggers"] = all_params["num_images"] all_params["num_images_per_trigger"] = 1 all_params["trigger_mode"] = TriggerMode.FREE_RUN - all_params["upper_left"] = Point3D(0, 0, 0) + all_params["upper_left"] = np.array([0, 0, 0]) return ArtemisParameters(**extract_artemis_params_from_flat_dict(all_params)) diff --git a/src/artemis/parameters/plan_specific/rotation_scan_internal_params.py b/src/artemis/parameters/plan_specific/rotation_scan_internal_params.py index c9081917c..22f544cad 100644 --- a/src/artemis/parameters/plan_specific/rotation_scan_internal_params.py +++ b/src/artemis/parameters/plan_specific/rotation_scan_internal_params.py @@ -2,6 +2,7 @@ from typing import Any, Optional +import numpy as np from dodal.devices.motors import XYZLimitBundle from dodal.devices.zebra import RotationDirection from dodal.parameters.experiment_parameter_base import AbstractExperimentParameterBase @@ -13,7 +14,6 @@ extract_artemis_params_from_flat_dict, extract_experiment_params_from_flat_dict, ) -from artemis.utils.utils import Point3D class RotationScanParams(BaseModel, AbstractExperimentParameterBase): @@ -80,12 +80,12 @@ def _preprocess_artemis_params( ): experiment_params: RotationScanParams = values["experiment_params"] all_params["num_images"] = experiment_params.get_num_images() - all_params["position"] = Point3D(*all_params["position"]) + all_params["position"] = np.array(all_params["position"]) if all_params["rotation_axis"] == "omega": all_params["omega_increment"] = all_params["rotation_increment"] else: all_params["omega_increment"] = 0 all_params["num_triggers"] = 1 all_params["num_images_per_trigger"] = all_params["num_images"] - all_params["upper_left"] = Point3D(*all_params["upper_left"]) + all_params["upper_left"] = np.array(all_params["upper_left"]) return ArtemisParameters(**extract_artemis_params_from_flat_dict(all_params)) From e307f6ff6973ab3e69d7aaf729846c8c99b9cdfc Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Jun 2023 10:31:13 +0100 Subject: [PATCH 21/22] fix ndarray serialisation for __eq__ and otherwise --- .../ispyb/ispyb_dataclass.py | 12 ++++++-- .../unit_tests/test_store_in_ispyb.py | 2 +- src/artemis/parameters/internal_parameters.py | 28 ------------------- .../tests/test_internal_parameters.py | 19 ++++++++++++- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py index c7eeade03..05a2a016a 100644 --- a/src/artemis/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/artemis/external_interaction/ispyb/ispyb_dataclass.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import numpy as np from pydantic import BaseModel, validator @@ -42,9 +42,15 @@ class Config: arbitrary_types_allowed = True json_encoders = {np.ndarray: lambda a: a.tolist()} + def dict(self, **kwargs): + as_dict = super().dict(**kwargs) + as_dict["upper_left"] = as_dict["upper_left"].tolist() + as_dict["position"] = as_dict["position"].tolist() + return as_dict + @validator("upper_left", pre=True) def _parse_upper_left( - cls, upper_left: list[int | float] | np.ndarray, values: dict[str, Any] + cls, upper_left: list[int | float] | np.ndarray, values: Dict[str, Any] ) -> np.ndarray: assert len(upper_left) == 3 if isinstance(upper_left, np.ndarray): @@ -53,7 +59,7 @@ def _parse_upper_left( @validator("position", pre=True) def _parse_position( - cls, position: list[int | float] | np.ndarray, values: dict[str, Any] + cls, position: list[int | float] | np.ndarray, values: Dict[str, Any] ) -> np.ndarray: assert len(position) == 3 if isinstance(position, np.ndarray): diff --git a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py index c092ddde0..ddf15ef2a 100644 --- a/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py +++ b/src/artemis/external_interaction/unit_tests/test_store_in_ispyb.py @@ -25,7 +25,7 @@ @pytest.fixture def dummy_params(): - dummy_params = FGSInternalParameters(default_raw_params()) + dummy_params = FGSInternalParameters(**default_raw_params()) dummy_params.artemis_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.artemis_params.ispyb_params.microns_per_pixel_x = 0.8 dummy_params.artemis_params.ispyb_params.microns_per_pixel_y = 0.8 diff --git a/src/artemis/parameters/internal_parameters.py b/src/artemis/parameters/internal_parameters.py index 527052f08..8c80c7c3e 100644 --- a/src/artemis/parameters/internal_parameters.py +++ b/src/artemis/parameters/internal_parameters.py @@ -50,34 +50,6 @@ class Config: **IspybParams.Config.json_encoders, } - def __repr__(self): - return ( - "artemis_params:\n" - f" zocalo_environment: {self.zocalo_environment}\n" - f" beamline: {self.beamline}\n" - f" insertion_prefix: {self.insertion_prefix}\n" - f" experiment_type: {self.experiment_type}\n" - f" detector_params: {self.detector_params}\n" - f" ispyb_params: {self.ispyb_params}\n" - ) - - def __eq__(self, other) -> bool: - if not isinstance(other, ArtemisParameters): - return NotImplemented - elif self.zocalo_environment != other.zocalo_environment: - return False - elif self.beamline != other.beamline: - return False - elif self.insertion_prefix != other.insertion_prefix: - return False - elif self.experiment_type != other.experiment_type: - return False - elif self.detector_params != other.detector_params: - return False - elif self.ispyb_params != other.ispyb_params: - return False - return True - def flatten_dict(d: dict, parent_items: dict = {}) -> dict: """Flatten a dictionary assuming all keys are unique.""" diff --git a/src/artemis/parameters/tests/test_internal_parameters.py b/src/artemis/parameters/tests/test_internal_parameters.py index 65464ee7c..67811b7c2 100644 --- a/src/artemis/parameters/tests/test_internal_parameters.py +++ b/src/artemis/parameters/tests/test_internal_parameters.py @@ -1,12 +1,16 @@ import copy import json +import numpy as np import pytest from dodal.devices.detector import DetectorParams from dodal.devices.fast_grid_scan import GridScanParams from pydantic import ValidationError -from artemis.external_interaction.ispyb.ispyb_dataclass import IspybParams +from artemis.external_interaction.ispyb.ispyb_dataclass import ( + ISPYB_PARAM_DEFAULTS, + IspybParams, +) from artemis.parameters import external_parameters from artemis.parameters.external_parameters import from_file from artemis.parameters.internal_parameters import ( @@ -45,6 +49,19 @@ def test_cant_initialise_abstract_internalparams(): ) +def test_ispyb_param_dict(): + ispyb_params = IspybParams(**ISPYB_PARAM_DEFAULTS) + as_dict = ispyb_params.dict() + assert isinstance(as_dict.get("position"), list) + modified_params = copy.deepcopy(ISPYB_PARAM_DEFAULTS) + modified_params["position"] = [123, 7777777, 3] + modified_ispyb_params = IspybParams(**modified_params) + assert ispyb_params != modified_ispyb_params + assert isinstance(modified_ispyb_params.position, np.ndarray) + modified_as_dict = modified_ispyb_params.dict() + assert modified_as_dict.get("position") == [123, 7777777, 3] + + def test_internal_param_serialisation_deserialisation(): data = from_file() internal_parameters = FGSInternalParameters(**data) From dfa5e689a9d98a953cabf41fa597e8357d1e93a8 Mon Sep 17 00:00:00 2001 From: David Perl Date: Fri, 2 Jun 2023 10:35:44 +0100 Subject: [PATCH 22/22] update dodal req --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8a8f2fb95..9a9ac56de 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,7 @@ install_requires = xarray doct databroker - dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@865a71cddc2bafd2378f512f6f2f8f2d1e8769bb + dodal @ git+https://github.com/DiamondLightSource/python-dodal.git@0570e5e6e5e134fd0697cadccc5bd6d2c6ed77d1 [options.extras_require]