From 8d933a68410150a0fbcba19587b612f629ab9f6e Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Wed, 19 Apr 2023 14:56:29 +0000 Subject: [PATCH 01/24] Add dodal git dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3f00faea07..d010a7043c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "fastapi[all]", "uvicorn", "requests", + "dodal @ git+https://git@github.com/DiamondLightSource/dodal", ] dynamic = ["version"] license.file = "LICENSE" From a5c1f82c0f42f2aad6753defceed4b28348bf1ec Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Wed, 26 Apr 2023 16:20:16 +0000 Subject: [PATCH 02/24] Add initial dodal device loading to context --- src/blueapi/core/context.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 9d28d4f6c3..0963119902 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -17,6 +17,7 @@ get_origin, get_type_hints, ) +from dodal.utils import make_all_devices from bluesky import RunEngine from pydantic import create_model @@ -103,6 +104,10 @@ def plan_2(...): self.plan(obj) def with_device_module(self, module: ModuleType) -> None: + if module.__package__ == "dodal": + devices = make_all_devices(module) + self.devices.update(devices) + for obj in load_module_all(module): if is_bluesky_compatible_device(obj): self.device(obj) From 1ab4c12164ea58ff4591324684706426383de767 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 27 Apr 2023 08:13:20 +0000 Subject: [PATCH 03/24] Allow only dodal style device modules --- src/blueapi/core/context.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 0963119902..accf0ecf0f 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -104,13 +104,7 @@ def plan_2(...): self.plan(obj) def with_device_module(self, module: ModuleType) -> None: - if module.__package__ == "dodal": - devices = make_all_devices(module) - self.devices.update(devices) - - for obj in load_module_all(module): - if is_bluesky_compatible_device(obj): - self.device(obj) + self.devices.update(make_all_devices(module)) def plan(self, plan: PlanGenerator) -> PlanGenerator: """ From d4502343949ef31ba78410870b270db38834354e Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 27 Apr 2023 15:59:35 +0000 Subject: [PATCH 04/24] Allow list of device and plan sources --- src/blueapi/config.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 3e7dce4570..7fa2327c6b 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Generic, Literal, Mapping, Type, TypeVar, Union +from typing import Any, Generic, Literal, Mapping, Type, TypeVar, TypedDict, Union import yaml from pydantic import BaseModel, Field, ValidationError, parse_obj_as @@ -9,7 +9,12 @@ LogLevel = Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -class StompConfig(BlueapiBaseModel): +class Source(TypedDict): + type: str + module: Union[Path, str] + + +class StompConfig(BaseModel): """ Config for connecting to stomp broker """ @@ -23,11 +28,20 @@ class EnvironmentConfig(BlueapiBaseModel): Config for the RunEngine environment """ - startup_script: Union[Path, str] = "blueapi.startup.example" + sources: list[Source] = [ + { + "type": "deviceFunctions", + "module": "blueapi.startup.example", + }, + { + "type": "planFunctions", + "module": "blueapi.plans", + }, + ] def __eq__(self, other: object) -> bool: if isinstance(other, EnvironmentConfig): - return str(self.startup_script) == str(other.startup_script) + return str(self.sources) == str(other.sources) return False From 2183e3cfbd0ff054dc641b69c82e4dc3ddeaa185 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 27 Apr 2023 16:17:09 +0000 Subject: [PATCH 05/24] Enable context to search multiple modules for devices and plans --- src/blueapi/core/context.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index accf0ecf0f..5ac7a83efe 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -71,9 +71,10 @@ def find_device(self, addr: Union[str, List[str]]) -> Optional[Device]: else: return find_component(self.devices, addr) - def with_startup_script(self, path: Union[Path, str]) -> None: - mod = import_module(str(path)) - self.with_module(mod) + def with_startup_script(self, paths: list[Union[Path, str]]) -> None: + for path in paths: + mod = import_module(str(path)) + self.with_module(mod) def with_module(self, module: ModuleType) -> None: self.with_plan_module(module) From ffd3e3bdb088e9b370afb0d8f39c36976c52a2a8 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Tue, 2 May 2023 10:38:23 +0000 Subject: [PATCH 06/24] Update to use dodal function format --- src/blueapi/startup/example.py | 98 +++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 31 deletions(-) diff --git a/src/blueapi/startup/example.py b/src/blueapi/startup/example.py index fbc27768e1..2244071d7e 100644 --- a/src/blueapi/startup/example.py +++ b/src/blueapi/startup/example.py @@ -1,38 +1,74 @@ from ophyd.sim import Syn2DGauss, SynGauss, SynSignal -from blueapi.plans import * # noqa: F401, F403 - from .simmotor import BrokenSynAxis, SynAxisWithMotionEvents -x = SynAxisWithMotionEvents(name="x", delay=1.0, events_per_move=8) -y = SynAxisWithMotionEvents(name="y", delay=3.0, events_per_move=24) -z = SynAxisWithMotionEvents(name="z", delay=2.0, events_per_move=16) -theta = SynAxisWithMotionEvents( - name="theta", delay=0.2, events_per_move=12, egu="degrees" -) -x_err = BrokenSynAxis(name="x_err", timeout=1.0) -sample_pressure = SynAxisWithMotionEvents( - name="sample_pressure", delay=30.0, events_per_move=128, egu="MPa", value=0.101 -) -sample_temperature = SynSignal( - func=lambda: ((x.position + y.position + z.position) / 1000.0) + 20.0, + +def x(name="x") -> SynAxisWithMotionEvents: + return SynAxisWithMotionEvents(name=name, delay=1.0, events_per_move=8) + + +def y(name="y") -> SynAxisWithMotionEvents: + return SynAxisWithMotionEvents(name=name, delay=3.0, events_per_move=24) + + +def z(name="z") -> SynAxisWithMotionEvents: + return SynAxisWithMotionEvents(name=name, delay=2.0, events_per_move=16) + + +def theta(name="theta") -> SynAxisWithMotionEvents: + return SynAxisWithMotionEvents( + name=name, delay=0.2, events_per_move=12, egu="degrees" + ) + + +def x_err(name="x_err") -> BrokenSynAxis: + return BrokenSynAxis(name=name, timeout=1.0) + + +def sample_pressure(name="sample_pressure") -> SynAxisWithMotionEvents: + return SynAxisWithMotionEvents( + name=name, delay=30.0, events_per_move=128, egu="MPa", value=0.101 + ) + + +def sample_temperature( + x: SynAxisWithMotionEvents, + y: SynAxisWithMotionEvents, + z: SynAxisWithMotionEvents, name="sample_temperature", -) -image_det = Syn2DGauss( +) -> SynSignal: + return SynSignal( + func=lambda: ((x.position + y.position + z.position) / 1000.0) + 20.0, + name=name, + ) + + +def image_det( + x: SynAxisWithMotionEvents, + y: SynAxisWithMotionEvents, name="image_det", - motor0=x, - motor_field0="x", - motor1=y, - motor_field1="y", - center=(0, 0), - Imax=1, - labels={"detectors"}, -) -current_det = SynGauss( +) -> Syn2DGauss: + return Syn2DGauss( + name=name, + motor0=x, + motor_field0="x", + motor1=y, + motor_field1="y", + center=(0, 0), + Imax=1, + labels={"detectors"}, + ) + + +def current_det( + x: SynAxisWithMotionEvents, name="current_det", - motor=x, - motor_field="x", - center=0.0, - Imax=1, - labels={"detectors"}, -) +) -> SynGauss: + return SynGauss( + name=name, + motor=x, + motor_field="x", + center=0.0, + Imax=1, + labels={"detectors"}, + ) From 8b5e2058ab519ae165dbc3a2c082e72b1d560a1a Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 4 May 2023 12:20:57 +0000 Subject: [PATCH 07/24] Add checking when adding devices to context --- src/blueapi/core/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 5ac7a83efe..ad5df2dc33 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -105,7 +105,8 @@ def plan_2(...): self.plan(obj) def with_device_module(self, module: ModuleType) -> None: - self.devices.update(make_all_devices(module)) + for device in make_all_devices(module).values(): + self.device(device) def plan(self, plan: PlanGenerator) -> PlanGenerator: """ From 50d0c759d58cb9143c58f8344fd0b0ed45bcc193 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 4 May 2023 12:36:15 +0000 Subject: [PATCH 08/24] Update p45 config to use dodal --- config/bl45p.yaml | 6 +- src/blueapi/startup/bl45p.py | 113 ----------------------------------- 2 files changed, 5 insertions(+), 114 deletions(-) delete mode 100644 src/blueapi/startup/bl45p.py diff --git a/config/bl45p.yaml b/config/bl45p.yaml index 9463d3e4a0..7d1bd1d0fa 100644 --- a/config/bl45p.yaml +++ b/config/bl45p.yaml @@ -1,2 +1,6 @@ env: - startupScript: blueapi.startup.bl45p + sources: + - type: dodal + module: dodal.p45 + - type: planFunctions + module: blueapi.plans diff --git a/src/blueapi/startup/bl45p.py b/src/blueapi/startup/bl45p.py deleted file mode 100644 index 6a8e5e3d78..0000000000 --- a/src/blueapi/startup/bl45p.py +++ /dev/null @@ -1,113 +0,0 @@ -from nslsii.ad33 import CamV33Mixin, SingleTriggerV33 -from ophyd import Component as Cpt -from ophyd import DetectorBase, EpicsMotor, MotorBundle -from ophyd.areadetector.base import ADComponent as Cpt -from ophyd.areadetector.cam import AreaDetectorCam -from ophyd.areadetector.detectors import DetectorBase -from ophyd.areadetector.filestore_mixins import FileStoreHDF5, FileStoreIterativeWrite -from ophyd.areadetector.plugins import HDF5Plugin - -from blueapi.plans import * # noqa: F401, F403 - - -class SampleY(MotorBundle): - top: EpicsMotor = Cpt(EpicsMotor, "Y:TOP") - bottom: EpicsMotor = Cpt(EpicsMotor, "Y:BOT") - - -class SampleTheta(MotorBundle): - top: EpicsMotor = Cpt(EpicsMotor, "THETA:TOP") - bottom: EpicsMotor = Cpt(EpicsMotor, "THETA:BOT") - - -class SampleStage(MotorBundle): - x: EpicsMotor = Cpt(EpicsMotor, "X") - y: SampleY = Cpt(SampleY, "") - theta: SampleTheta = Cpt(SampleTheta, "") - - -class Choppers(MotorBundle): - x: EpicsMotor = Cpt(EpicsMotor, "ENDAT") - y: EpicsMotor = Cpt(EpicsMotor, "BISS") - - -_ACQUIRE_BUFFER_PERIOD = 0.2 - - -class NonBlockingCam(AreaDetectorCam, CamV33Mixin): - ... - - -# define a detector device class that has the correct PV suffixes for the rigs -class GigeCamera(SingleTriggerV33, DetectorBase): - class HDF5File(HDF5Plugin, FileStoreHDF5, FileStoreIterativeWrite): - pool_max_buffers = None - file_number_sync = None - file_number_write = None - - def get_frames_per_point(self): - return self.parent.cam.num_images.get() - - cam: NonBlockingCam = Cpt(NonBlockingCam, suffix="DET:") - hdf: HDF5File = Cpt( - HDF5File, - suffix="HDF5:", - root="/dls/tmp/vid18871/data", - write_path_template="%Y", - ) - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.hdf.kind = "normal" - - # Get stage to wire up the plugins - self.stage_sigs[self.hdf.nd_array_port] = self.cam.port_name.get() - - # Makes the detector allow non-blocking AD plugins but makes Ophyd use - # the AcquireBusy PV to determine when an acquisition is complete - self.cam.ensure_nonblocking() - - # Reset array counter on stage - self.stage_sigs[self.cam.array_counter] = 0 - - # Set image mode to multiple on stage so we have the option, can still - # set num_images to 1 - self.stage_sigs[self.cam.image_mode] = "Multiple" - - # For now, this Ophyd device does not support hardware - # triggered scanning, disable on stage - self.stage_sigs[self.cam.trigger_mode] = "Off" - - def make_data_key(self): - source = "PV:{}".format(self.prefix) - # This shape is expected to match arr.shape for the array. - shape = ( - self.cam.num_images.get(), - self.cam.array_size.array_size_y.get(), - self.cam.array_size.array_size_x.get(), - ) - return dict(shape=shape, source=source, dtype="array", external="FILESTORE:") - - def stage(self, *args, **kwargs): - # We have to manually set the acquire period bcause the EPICS driver - # doesn't do it for us. If acquire time is a staged signal, we use the - # stage value to calculate the acquire period, otherwise we perform - # a caget and use the current acquire time. - if self.cam.acquire_time in self.stage_sigs: - acquire_time = self.stage_sigs[self.cam.acquire_time] - else: - acquire_time = self.cam.acquire_time.get() - acquire_period = acquire_time + _ACQUIRE_BUFFER_PERIOD - self.stage_sigs[self.cam.acquire_period] = acquire_period - - # Now calling the super method should set the acquire period - super(GigeCamera, self).stage(*args, **kwargs) - - -sample = SampleStage(name="sample", prefix="BL45P-MO-STAGE-01:") -choppers = Choppers(name="chopper", prefix="BL45P-MO-CHOP-01:") -det = GigeCamera(name="det", prefix="BL45P-EA-MAP-01:") -diff = GigeCamera(name="diff", prefix="BL45P-EA-DIFF-01:") - -for device in sample, choppers, det, diff: - device.wait_for_connection() # type: ignore From 913990b2614002d2ee7b0707d334c1cbb555aaf9 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 4 May 2023 12:37:51 +0000 Subject: [PATCH 09/24] Update adsim.yaml to use multiple sources --- config/adsim.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/adsim.yaml b/config/adsim.yaml index 83a41f29fd..7a343e6b3b 100644 --- a/config/adsim.yaml +++ b/config/adsim.yaml @@ -1,2 +1,6 @@ env: - startupScript: blueapi.startup.adsim + sources: + - type: deviceFunctions + module: blueapi.startup.adsim + - type: planFunctions + module: blueapi.startup.adsim From 49a0e554734bdeaa4745ea445b5e5a1a8488488f Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 4 May 2023 12:38:26 +0000 Subject: [PATCH 10/24] Update worker config for multiple sources --- helm/blueapi/values.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index 14110c59d7..6b894fc3e2 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -59,7 +59,11 @@ affinity: {} worker: env: - startupScript: blueapi.startup.example + sources: + - type: deviceFunctions + module: blueapi.startup.example + - type: planFunctions + module: blueapi.plans stomp: host: activemq port: 61613 From db00d285520c4f46f86c6f08f8413f001409fde2 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 5 May 2023 08:38:12 +0000 Subject: [PATCH 11/24] Add module loading tests for context --- tests/core/fake_device_module.py | 13 +++++++++++++ tests/core/fake_plan_module.py | 1 + tests/core/test_context.py | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 tests/core/fake_device_module.py create mode 100644 tests/core/fake_plan_module.py diff --git a/tests/core/fake_device_module.py b/tests/core/fake_device_module.py new file mode 100644 index 0000000000..3f6b3eb07c --- /dev/null +++ b/tests/core/fake_device_module.py @@ -0,0 +1,13 @@ +from unittest.mock import MagicMock + +from ophyd import EpicsMotor + + +def fake_motor() -> EpicsMotor: + return _mock_with_name("motor") + + +def _mock_with_name(name: str) -> MagicMock: + mock = MagicMock() + mock.name = name + return mock diff --git a/tests/core/fake_plan_module.py b/tests/core/fake_plan_module.py new file mode 100644 index 0000000000..89a9132d27 --- /dev/null +++ b/tests/core/fake_plan_module.py @@ -0,0 +1 @@ +from blueapi.plans import scan # noqa: F403 diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 059a7473ac..e2f9f9e991 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -109,6 +109,13 @@ def test_add_invalid_plan(empty_context: BlueskyContext, plan: PlanGenerator) -> empty_context.plan(plan) +def test_add_plan_from_module(empty_context: BlueskyContext) -> None: + import tests.core.fake_plan_module as plan_module + + empty_context.with_plan_module(plan_module) + assert {"scan"} == empty_context.plans.keys() + + def test_add_named_device(empty_context: BlueskyContext, sim_motor: SynAxis) -> None: empty_context.device(sim_motor) assert empty_context.devices[SIM_MOTOR_NAME] is sim_motor @@ -136,6 +143,13 @@ def test_override_device_name( assert empty_context.devices["foo"] is sim_motor +def test_add_devices_from_module(empty_context: BlueskyContext) -> None: + import tests.core.fake_device_module as device_module + + empty_context.with_device_module(device_module) + assert {"motor"} == empty_context.devices.keys() + + @pytest.mark.parametrize( "addr", ["sim", "sim_det", "sim.setpoint", ["sim"], ["sim", "setpoint"]] ) @@ -169,6 +183,16 @@ def test_add_non_device(empty_context: BlueskyContext) -> None: empty_context.device("not a device") # type: ignore +def test_add_devices_and_plans_from_modules_with_startup_script( + empty_context: BlueskyContext, +) -> None: + empty_context.with_startup_script( + ["tests.core.fake_device_module", "tests.core.fake_plan_module"] + ) + assert {"motor"} == empty_context.devices.keys() + assert {"scan"} == empty_context.plans.keys() + + def test_function_spec(empty_context: BlueskyContext) -> None: spec = empty_context._type_spec_for_function(has_some_params) assert spec == {"foo": (int, 42), "bar": (str, "bar")} From ae2d1db7a2c1939e914d99478e9f09b2f0ac6502 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 5 May 2023 08:39:31 +0000 Subject: [PATCH 12/24] Correct ignore pre-commit comment --- tests/core/fake_plan_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/fake_plan_module.py b/tests/core/fake_plan_module.py index 89a9132d27..450bc58310 100644 --- a/tests/core/fake_plan_module.py +++ b/tests/core/fake_plan_module.py @@ -1 +1 @@ -from blueapi.plans import scan # noqa: F403 +from blueapi.plans import scan # noqa: F401 From 1ada7d97691c997b2d9efaa7c971d6658cc59a78 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 5 May 2023 10:34:18 +0000 Subject: [PATCH 13/24] Make module name qualification explicit --- pyproject.toml | 1 + tests/__init__.py | 0 tests/utils/test_modules.py | 4 ++-- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 tests/__init__.py diff --git a/pyproject.toml b/pyproject.toml index d010a7043c..8109ddf844 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ write_to = "src/blueapi/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules +namespace_packages = false # rely only on __init__ files to determine fully qualified module names. [tool.isort] float_to_top = true diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/utils/test_modules.py b/tests/utils/test_modules.py index 429f035e46..2a33cc17a3 100644 --- a/tests/utils/test_modules.py +++ b/tests/utils/test_modules.py @@ -4,10 +4,10 @@ def test_imports_all(): - module = import_module(".hasall", package="utils") + module = import_module(".hasall", package="tests.utils") assert list(load_module_all(module)) == ["hello", 9] def test_imports_everything_without_all(): - module = import_module(".lacksall", package="utils") + module = import_module(".lacksall", package="tests.utils") assert list(load_module_all(module)) == [3, "hello", 9] From ea128e3373866189cac5e3f7a9ba1fa7eed4ba2a Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 5 May 2023 12:43:41 +0000 Subject: [PATCH 14/24] Swap typedict for basemodel in config --- src/blueapi/config.py | 14 ++++---------- src/blueapi/core/context.py | 10 +++++----- src/blueapi/service/handler.py | 2 +- tests/core/test_context.py | 12 +++++++++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 7fa2327c6b..e6ae62ada1 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Generic, Literal, Mapping, Type, TypeVar, TypedDict, Union +from typing import Any, Generic, Literal, Mapping, Type, TypeVar, Union import yaml from pydantic import BaseModel, Field, ValidationError, parse_obj_as @@ -9,7 +9,7 @@ LogLevel = Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -class Source(TypedDict): +class Source(BaseModel): type: str module: Union[Path, str] @@ -29,14 +29,8 @@ class EnvironmentConfig(BlueapiBaseModel): """ sources: list[Source] = [ - { - "type": "deviceFunctions", - "module": "blueapi.startup.example", - }, - { - "type": "planFunctions", - "module": "blueapi.plans", - }, + Source(type="deviceFunctions", module="blueapi.startup.example"), + Source(type="planFunctions", module="blueapi.plans"), ] def __eq__(self, other: object) -> bool: diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index ad5df2dc33..0961700572 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from importlib import import_module from inspect import Parameter, signature -from pathlib import Path from types import ModuleType from typing import ( Any, @@ -17,11 +16,12 @@ get_origin, get_type_hints, ) -from dodal.utils import make_all_devices from bluesky import RunEngine +from dodal.utils import make_all_devices from pydantic import create_model +from blueapi.config import EnvironmentConfig from blueapi.utils import BlueapiPlanModelConfig, load_module_all from .bluesky_types import ( @@ -71,9 +71,9 @@ def find_device(self, addr: Union[str, List[str]]) -> Optional[Device]: else: return find_component(self.devices, addr) - def with_startup_script(self, paths: list[Union[Path, str]]) -> None: - for path in paths: - mod = import_module(str(path)) + def with_config(self, config: EnvironmentConfig) -> None: + for source in config.sources: + mod = import_module(str(source.module)) self.with_module(mod) def with_module(self, module: ModuleType) -> None: diff --git a/src/blueapi/service/handler.py b/src/blueapi/service/handler.py index f99a7aee62..221e7e791f 100644 --- a/src/blueapi/service/handler.py +++ b/src/blueapi/service/handler.py @@ -21,7 +21,7 @@ def __init__(self, config: Optional[ApplicationConfig] = None) -> None: logging.basicConfig(level=self.config.logging.level) - self.context.with_startup_script(self.config.env.startup_script) + self.context.with_config(self.config.env) self.worker = RunEngineWorker(self.context) self.message_bus = StompMessagingTemplate.autoconfigured(self.config.stomp) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index e2f9f9e991..8c443d8699 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -6,6 +6,7 @@ from bluesky.protocols import Descriptor, Movable, Readable, Reading, SyncOrAsync from ophyd.sim import SynAxis, SynGauss +from blueapi.config import EnvironmentConfig, Source from blueapi.core import ( BlueskyContext, MsgGenerator, @@ -183,11 +184,16 @@ def test_add_non_device(empty_context: BlueskyContext) -> None: empty_context.device("not a device") # type: ignore -def test_add_devices_and_plans_from_modules_with_startup_script( +def test_add_devices_and_plans_from_modules_with_config( empty_context: BlueskyContext, ) -> None: - empty_context.with_startup_script( - ["tests.core.fake_device_module", "tests.core.fake_plan_module"] + empty_context.with_config( + EnvironmentConfig( + sources=[ + Source(type="deviceFunctions", module="tests.core.fake_device_module"), + Source(type="planFunctions", module="tests.core.fake_plan_module"), + ] + ) ) assert {"motor"} == empty_context.devices.keys() assert {"scan"} == empty_context.plans.keys() From 637130615fb7035c1eb1ae984c3464de82b88da9 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 5 May 2023 15:24:41 +0000 Subject: [PATCH 15/24] Test device functions with dependencies --- tests/core/fake_device_module.py | 22 ++++++++++++++++++++-- tests/core/test_context.py | 14 ++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/tests/core/fake_device_module.py b/tests/core/fake_device_module.py index 3f6b3eb07c..feaafac558 100644 --- a/tests/core/fake_device_module.py +++ b/tests/core/fake_device_module.py @@ -3,8 +3,26 @@ from ophyd import EpicsMotor -def fake_motor() -> EpicsMotor: - return _mock_with_name("motor") +def fake_motor_bundle_b( + fake_motor_x: EpicsMotor, + fake_motor_y: EpicsMotor, +) -> EpicsMotor: + return _mock_with_name("motor_bundle_b") + + +def fake_motor_x() -> EpicsMotor: + return _mock_with_name("motor_x") + + +def fake_motor_y() -> EpicsMotor: + return _mock_with_name("motor_y") + + +def fake_motor_bundle_a( + fake_motor_x: EpicsMotor, + fake_motor_y: EpicsMotor, +) -> EpicsMotor: + return _mock_with_name("motor_bundle_a") def _mock_with_name(name: str) -> MagicMock: diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 8c443d8699..3b9dd7c49d 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -148,7 +148,12 @@ def test_add_devices_from_module(empty_context: BlueskyContext) -> None: import tests.core.fake_device_module as device_module empty_context.with_device_module(device_module) - assert {"motor"} == empty_context.devices.keys() + assert { + "motor_x", + "motor_y", + "motor_bundle_a", + "motor_bundle_b", + } == empty_context.devices.keys() @pytest.mark.parametrize( @@ -195,7 +200,12 @@ def test_add_devices_and_plans_from_modules_with_config( ] ) ) - assert {"motor"} == empty_context.devices.keys() + assert { + "motor_x", + "motor_y", + "motor_bundle_a", + "motor_bundle_b", + } == empty_context.devices.keys() assert {"scan"} == empty_context.plans.keys() From 034851db33b3e3644eacf5f3e8ffa2314d2c300a Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 14:06:20 +0000 Subject: [PATCH 16/24] Make source type an enum --- src/blueapi/config.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/blueapi/config.py b/src/blueapi/config.py index e6ae62ada1..e1fefa080a 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -1,3 +1,4 @@ +from enum import Enum from pathlib import Path from typing import Any, Generic, Literal, Mapping, Type, TypeVar, Union @@ -9,8 +10,14 @@ LogLevel = Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] +class SourceKind(str, Enum): + planFunctions = "planFunctions" + deviceFunctions = "deviceFunctions" + dodal = "dodal" + + class Source(BaseModel): - type: str + kind: SourceKind module: Union[Path, str] @@ -29,8 +36,8 @@ class EnvironmentConfig(BlueapiBaseModel): """ sources: list[Source] = [ - Source(type="deviceFunctions", module="blueapi.startup.example"), - Source(type="planFunctions", module="blueapi.plans"), + Source(kind=SourceKind.deviceFunctions, module="blueapi.startup.example"), + Source(kind=SourceKind.planFunctions, module="blueapi.plans"), ] def __eq__(self, other: object) -> bool: From 8ea37597eff2d97efab4f146ed85861edf487f29 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 14:32:04 +0000 Subject: [PATCH 17/24] Allocate device and plan adding based on source kind --- src/blueapi/core/context.py | 20 +++++++++++++------- tests/core/test_context.py | 11 ++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 0961700572..312239d03f 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -18,12 +18,11 @@ ) from bluesky import RunEngine -from dodal.utils import make_all_devices +from bluesky.protocols import Flyable, Readable from pydantic import create_model -from blueapi.config import EnvironmentConfig +from blueapi.config import EnvironmentConfig, SourceKind from blueapi.utils import BlueapiPlanModelConfig, load_module_all - from .bluesky_types import ( BLUESKY_PROTOCOLS, Device, @@ -74,11 +73,13 @@ def find_device(self, addr: Union[str, List[str]]) -> Optional[Device]: def with_config(self, config: EnvironmentConfig) -> None: for source in config.sources: mod = import_module(str(source.module)) - self.with_module(mod) - def with_module(self, module: ModuleType) -> None: - self.with_plan_module(module) - self.with_device_module(module) + if source.kind is SourceKind.planFunctions: + self.with_plan_module(mod) + elif source.kind is SourceKind.deviceFunctions: + self.with_device_module(mod) + elif source.kind is SourceKind.dodal: + self.with_dodal_module(mod) def with_plan_module(self, module: ModuleType) -> None: """ @@ -105,6 +106,11 @@ def plan_2(...): self.plan(obj) def with_device_module(self, module: ModuleType) -> None: + self.with_dodal_module(module) + + def with_dodal_module(self, module: ModuleType) -> None: + from dodal.utils import make_all_devices + for device in make_all_devices(module).values(): self.device(device) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 3b9dd7c49d..e6185a76e4 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -6,7 +6,7 @@ from bluesky.protocols import Descriptor, Movable, Readable, Reading, SyncOrAsync from ophyd.sim import SynAxis, SynGauss -from blueapi.config import EnvironmentConfig, Source +from blueapi.config import EnvironmentConfig, Source, SourceKind from blueapi.core import ( BlueskyContext, MsgGenerator, @@ -195,8 +195,13 @@ def test_add_devices_and_plans_from_modules_with_config( empty_context.with_config( EnvironmentConfig( sources=[ - Source(type="deviceFunctions", module="tests.core.fake_device_module"), - Source(type="planFunctions", module="tests.core.fake_plan_module"), + Source( + kind=SourceKind.deviceFunctions, + module="tests.core.fake_device_module", + ), + Source( + kind=SourceKind.planFunctions, module="tests.core.fake_plan_module" + ), ] ) ) From 8c984d4504403cf3af2c77268e1d829130e724da Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 14:32:42 +0000 Subject: [PATCH 18/24] Change source type to kind --- config/adsim.yaml | 4 ++-- config/bl45p.yaml | 4 ++-- helm/blueapi/values.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/adsim.yaml b/config/adsim.yaml index 7a343e6b3b..c3b3196f42 100644 --- a/config/adsim.yaml +++ b/config/adsim.yaml @@ -1,6 +1,6 @@ env: sources: - - type: deviceFunctions + - kind: deviceFunctions module: blueapi.startup.adsim - - type: planFunctions + - kind: planFunctions module: blueapi.startup.adsim diff --git a/config/bl45p.yaml b/config/bl45p.yaml index 7d1bd1d0fa..0699c52ea2 100644 --- a/config/bl45p.yaml +++ b/config/bl45p.yaml @@ -1,6 +1,6 @@ env: sources: - - type: dodal + - kind: dodal module: dodal.p45 - - type: planFunctions + - kind: planFunctions module: blueapi.plans diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index 6b894fc3e2..7f9c8de102 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -60,9 +60,9 @@ affinity: {} worker: env: sources: - - type: deviceFunctions + - kind: deviceFunctions module: blueapi.startup.example - - type: planFunctions + - kind: planFunctions module: blueapi.plans stomp: host: activemq From 9103513f14648d76dd8749fc8aec0e7fe447309d Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 15:41:30 +0000 Subject: [PATCH 19/24] Remove unused imports --- src/blueapi/core/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 312239d03f..6b3e3a3659 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -18,11 +18,11 @@ ) from bluesky import RunEngine -from bluesky.protocols import Flyable, Readable from pydantic import create_model from blueapi.config import EnvironmentConfig, SourceKind from blueapi.utils import BlueapiPlanModelConfig, load_module_all + from .bluesky_types import ( BLUESKY_PROTOCOLS, Device, From ad6aecd5032698257005094ce6a9a0177e681ca4 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 16:04:23 +0000 Subject: [PATCH 20/24] Make test context use default environment config --- tests/worker/test_reworker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/worker/test_reworker.py b/tests/worker/test_reworker.py index 3f53b2a0ea..458b0fe88a 100644 --- a/tests/worker/test_reworker.py +++ b/tests/worker/test_reworker.py @@ -4,6 +4,7 @@ import pytest +from blueapi.config import EnvironmentConfig from blueapi.core import BlueskyContext, EventStream from blueapi.worker import ( RunEngineWorker, @@ -20,7 +21,7 @@ @pytest.fixture def context() -> BlueskyContext: ctx = BlueskyContext() - ctx.with_startup_script("blueapi.startup.example") + ctx.with_config(EnvironmentConfig()) return ctx From adac983138aa3255502ab0dd7386f31748c3982c Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Thu, 11 May 2023 16:04:41 +0000 Subject: [PATCH 21/24] Update source format --- config/defaults.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/defaults.yaml b/config/defaults.yaml index 32d1a3f273..234ac0d708 100644 --- a/config/defaults.yaml +++ b/config/defaults.yaml @@ -1,5 +1,9 @@ env: - startupScript: blueapi.startup.example + sources: + - kind: deviceFunctions + module: blueapi.startup.example + - kind: planFunctions + module: blueapi.plans stomp: host: localhost port: 61613 From 5f069f8394cb56105ac7104d70e2306a5259b908 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Fri, 12 May 2023 09:28:11 +0000 Subject: [PATCH 22/24] Update enum case to abide by pep8 --- src/blueapi/config.py | 10 +++++----- src/blueapi/core/context.py | 6 +++--- tests/core/test_context.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/blueapi/config.py b/src/blueapi/config.py index e1fefa080a..7c85294b93 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -11,9 +11,9 @@ class SourceKind(str, Enum): - planFunctions = "planFunctions" - deviceFunctions = "deviceFunctions" - dodal = "dodal" + PLAN_FUNCTIONS = "planFunctions" + DEVICE_FUNCTIONS = "deviceFunctions" + DODAL = "dodal" class Source(BaseModel): @@ -36,8 +36,8 @@ class EnvironmentConfig(BlueapiBaseModel): """ sources: list[Source] = [ - Source(kind=SourceKind.deviceFunctions, module="blueapi.startup.example"), - Source(kind=SourceKind.planFunctions, module="blueapi.plans"), + Source(kind=SourceKind.DEVICE_FUNCTIONS, module="blueapi.startup.example"), + Source(kind=SourceKind.PLAN_FUNCTIONS, module="blueapi.plans"), ] def __eq__(self, other: object) -> bool: diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index 6b3e3a3659..31c8317be4 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -74,11 +74,11 @@ def with_config(self, config: EnvironmentConfig) -> None: for source in config.sources: mod = import_module(str(source.module)) - if source.kind is SourceKind.planFunctions: + if source.kind is SourceKind.PLAN_FUNCTIONS: self.with_plan_module(mod) - elif source.kind is SourceKind.deviceFunctions: + elif source.kind is SourceKind.DEVICE_FUNCTIONS: self.with_device_module(mod) - elif source.kind is SourceKind.dodal: + elif source.kind is SourceKind.DODAL: self.with_dodal_module(mod) def with_plan_module(self, module: ModuleType) -> None: diff --git a/tests/core/test_context.py b/tests/core/test_context.py index e6185a76e4..bbdeb79ad5 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -196,11 +196,11 @@ def test_add_devices_and_plans_from_modules_with_config( EnvironmentConfig( sources=[ Source( - kind=SourceKind.deviceFunctions, + kind=SourceKind.DEVICE_FUNCTIONS, module="tests.core.fake_device_module", ), Source( - kind=SourceKind.planFunctions, module="tests.core.fake_plan_module" + kind=SourceKind.PLAN_FUNCTIONS, module="tests.core.fake_plan_module" ), ] ) From 7b03b80224a42d5606d594869f69da5ab5c7d274 Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Mon, 15 May 2023 11:47:26 +0000 Subject: [PATCH 23/24] Update dodal dependency format --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8109ddf844..0d83bb954a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "fastapi[all]", "uvicorn", "requests", - "dodal @ git+https://git@github.com/DiamondLightSource/dodal", + "dodal @ git+https://github.com/DiamondLightSource/dodal.git", + ] dynamic = ["version"] license.file = "LICENSE" From c4ef315b1f016b916ec8867ae8fe398d1854bdde Mon Sep 17 00:00:00 2001 From: Abigail Emery Date: Mon, 15 May 2023 13:11:22 +0000 Subject: [PATCH 24/24] Exclude dodal from lockfile Avoids current skeleton bug, https://github.com/DiamondLightSource/python3-pip-skeleton/issues/127. --- .github/actions/install_requirements/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 25a146d164..aea1e0d4b8 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -30,7 +30,7 @@ runs: - name: Create lockfile run: | mkdir -p lockfiles - pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} + pip freeze --exclude-editable --exclude dodal > lockfiles/${{ inputs.requirements_file }} # delete the self referencing line and make sure it isn't blank sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} shell: bash @@ -55,4 +55,3 @@ runs: fi fi shell: bash -