HICAT-994 Device servers using Python shared memory managers

.github/workflows/CI.yml

@@ -12,7 +12,7 @@ jobs:
     name: flake8 (syntax only) on Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 2
+      max-parallel: 1
       matrix:
         python-version: [3.7]
     env:

catkit/conftest.py

@@ -3,11 +3,17 @@
 import pytest
 
 from catkit.testbed import devices
+from catkit.testbed.caching import DeviceCacheEnum
 import catkit.util
+from catkit.multiprocessing import EXCEPTION_SERVER_ADDRESS, SharedMemoryManager
 
 catkit.util.simulation = True
 
 
+def pytest_configure(config):
+    config.addinivalue_line("markers", "dont_own_exception_handler: Neither start nor shutdown the exception handler server.")
+
+
 @pytest.fixture(scope="function", autouse=False)
 def derestricted_device_cache():
     # Setup.
@@ -18,3 +24,18 @@ def derestricted_device_cache():
         devices["npoint_a"]
     # Teardown.
     gc.collect()
+
+
+@pytest.fixture(scope="function", autouse=True)
+def exception_handler(request):
+    if "dont_own_exception_handler" not in request.keywords:
+        with SharedMemoryManager(address=EXCEPTION_SERVER_ADDRESS):
+            yield
+    else:
+        yield
+
+
+@pytest.fixture(scope="function", autouse=True)
+def clear_lru_cache():
+    DeviceCacheEnum.get_device.cache_clear()
+    DeviceCacheEnum._missing_.cache_clear()

catkit/datalogging/data_log_io.py

@@ -215,6 +215,7 @@ def value(self, value):
         self._serialized_length = None
         self._log_dir = None
 
+
 class DataLogWriter(object):
     '''A writer to write events to a binary log file.
 
@@ -240,6 +241,8 @@ def __init__(self, log_dir, flush_every=10, index_fname=None):
         if index_fname is None:
             index_fname = _INDEX_FNAME
 
+        os.makedirs(self.log_dir, exist_ok=True)
+
         self.index_path = os.path.join(log_dir, index_fname)
 
         if os.path.exists(self.index_path):
@@ -253,6 +256,14 @@ def __init__(self, log_dir, flush_every=10, index_fname=None):
         self._n = 0
         self._closed = False
 
+    def __enter__(self):
+        # TODO: Should/can this ADD itself to catkit.datalogging?
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # TODO: Should/can this REMOVE itself to catkit.datalogging?
+        return self.close()
+
     def log(self, wall_time, tag, value, value_type):
         '''Add an event to the log file.
 
@@ -331,6 +342,7 @@ def close(self):
 
         self._closed = True
 
+
 class DataLogReader(object):
     '''A reader for data log files produced by `DataLogWriter`.
 
@@ -359,6 +371,12 @@ def __init__(self, log_dir, load_in_memory=False, index_fname=None):
 
         self.reload()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self.close()
+
     def reload(self, force=False):
         '''Reload the data log from disk.
 

catkit/datalogging/tests/test_datalogging.py

@@ -6,86 +6,68 @@
 
 import catkit.datalogging
 
-def test_data_log_interface():
+
+def test_data_log_interface(tmpdir):
     logger = catkit.datalogging.get_logger(__name__)
 
     # Make sure this doesn't crash, even though nothing should be written out.
     logger.log_scalar('tag', 5)
 
-    log_dir = './data_log_test'
-    if os.path.exists(log_dir):
-        shutil.rmtree(log_dir)
-    os.makedirs(log_dir)
-
-    writer = catkit.datalogging.DataLogWriter(log_dir)
-
-    logger.log_scalar('tag2', 10)
+    with catkit.datalogging.DataLogWriter(tmpdir):
+        logger.log_scalar('tag2', 10)
 
-    # Cleanup
-    writer.close()
-    shutil.rmtree(log_dir)
 
-def test_data_log_retrieval():
+def test_data_log_retrieval(tmpdir):
     logger = catkit.datalogging.get_logger(__name__)
 
-    log_dir = './data_log_test'
-    if os.path.exists(log_dir):
-        shutil.rmtree(log_dir)
-    os.makedirs(log_dir)
-
-    writer = catkit.datalogging.DataLogWriter(log_dir)
-    catkit.datalogging.DataLogger.add_writer(writer)
-
-    scalar = float(np.random.randn(1))
-    tensor = np.random.randn(100, 250)
-    curve_x = np.random.randn(30)
-    curve_y = np.random.randn(30)
+    with catkit.datalogging.DataLogWriter(tmpdir) as writer:
+        catkit.datalogging.DataLogger.add_writer(writer)
 
-    plt.plot(curve_x, curve_y)
+        scalar = float(np.random.randn(1))
+        tensor = np.random.randn(100, 250)
+        curve_x = np.random.randn(30)
+        curve_y = np.random.randn(30)
 
-    hdu = fits.PrimaryHDU(tensor)
-    fits_fname = os.path.join(log_dir, 'tensor.fits')
-    hdu.writeto(fits_fname)
+        plt.plot(curve_x, curve_y)
 
-    logger.log_scalar('a', scalar)
-    logger.log_scalar('a', scalar * 2)
-    logger.log_scalar('a', scalar * -0.5)
+        hdu = fits.PrimaryHDU(tensor)
+        fits_fname = os.path.join(tmpdir, 'tensor.fits')
+        hdu.writeto(fits_fname)
 
-    logger.log_tensor('b', tensor)
+        logger.log_scalar('a', scalar)
+        logger.log_scalar('a', scalar * 2)
+        logger.log_scalar('a', scalar * -0.5)
 
-    logger.log_curve('c', curve_x, curve_y)
+        logger.log_tensor('b', tensor)
 
-    logger.log_figure('d')
+        logger.log_curve('c', curve_x, curve_y)
 
-    logger.log_fits_file('e', fits_fname)
+        logger.log_figure('d')
 
-    # Unregister writer
-    catkit.datalogging.DataLogger.remove_writer(writer)
-    writer.close()
+        logger.log_fits_file('e', fits_fname)
 
-    reader = catkit.datalogging.DataLogReader(log_dir)
+        # Unregister writer
+        catkit.datalogging.DataLogger.remove_writer(writer)
 
-    wall_time, scalars = reader.get('a')
-    assert np.allclose(scalars[0], scalar)
-    assert len(scalars) == 3
+    with catkit.datalogging.DataLogReader(tmpdir) as reader:
 
-    wall_time, scalars = reader.get('a', slice(1,None))
-    assert len(scalars) == 2
-    assert len(wall_time) == 2
+        wall_time, scalars = reader.get('a')
+        assert np.allclose(scalars[0], scalar)
+        assert len(scalars) == 3
 
-    wall_time, tensors = reader.get('b')
-    assert np.allclose(tensors[0], tensor)
+        wall_time, scalars = reader.get('a', slice(1,None))
+        assert len(scalars) == 2
+        assert len(wall_time) == 2
 
-    wall_time, curve = reader.get('c')
-    assert np.allclose(curve[0]['x'], curve_x)
-    assert np.allclose(curve[0]['y'], curve_y)
+        wall_time, tensors = reader.get('b')
+        assert np.allclose(tensors[0], tensor)
 
-    wall_time, figs = reader.get('d')
-    assert figs[0].ndim == 3
+        wall_time, curve = reader.get('c')
+        assert np.allclose(curve[0]['x'], curve_x)
+        assert np.allclose(curve[0]['y'], curve_y)
 
-    wall_time, fits_files = reader.get('e')
-    assert np.allclose(fits_files[0][0].data, tensor)
+        wall_time, figs = reader.get('d')
+        assert figs[0].ndim == 3
 
-    # Cleanup
-    reader.close()
-    shutil.rmtree(log_dir)
+        wall_time, fits_files = reader.get('e')
+        assert np.allclose(fits_files[0][0].data, tensor)

catkit/emulators/ZwoCamera.py

@@ -27,7 +27,7 @@ def get_camera_mappings(cls):
         return camera_mappings
 
     def __init__(self, config_id):
-        self.log = logging.getLogger(__name__)
+        self.log = logging.getLogger()
 
         self.config_id = config_id
         self.image_type = None

catkit/emulators/boston_dm.py

@@ -1,17 +1,18 @@
 import copy
 import logging
 
+from multiprocess.managers import BaseProxy
 import numpy as np
-
 import poppy.dms
 
 import catkit.util
 import catkit.hardware.boston.DmCommand
 from catkit.hardware.boston.BostonDmController import BostonDmController
 from catkit.interfaces.Instrument import SimInstrument
+from catkit.multiprocessing import MutexedNamespace
 
 
-class PoppyBostonDM(poppy.dms.ContinuousDeformableMirror):
+class PoppyBostonDM(MutexedNamespace, poppy.dms.ContinuousDeformableMirror):
     """
     Wraps `poppy.dms.ContinuousDeformableMirror` so as to encapsulate the additional DM
     attributes required to describe a typical Boston Micromachines DM.
@@ -52,17 +53,17 @@ class PoppyBmcEmulator:
     NO_ERR = 0
 
     def __init__(self, num_actuators, command_length, dac_bit_width, dm1, dm2=None):
-        self.log = logging.getLogger(f"{self.__module__}.{self.__class__.__qualname__}")
+        self.log = logging.getLogger()
         self._num_actuators = num_actuators
         self._command_length = command_length
         self._dac_bit_width = dac_bit_width
         self.dm1 = dm1
         self.dm2 = dm2
 
         # As the class name suggests, the design only works with ``poppy.dms.ContinuousDeformableMirror``.
-        assert isinstance(dm1, PoppyBostonDM)
+        assert isinstance(dm1, (PoppyBostonDM, BaseProxy)), type(dm1)
         if dm2 is not None:
-            assert isinstance(dm2, PoppyBostonDM)
+            assert isinstance(dm2, (PoppyBostonDM, BaseProxy)), type(dm2)
 
     def BmcDm(self):
         return self

catkit/emulators/iris_ao_controller.py

@@ -1,6 +1,7 @@
 import os
 
 import astropy.units as u
+from multiprocess.managers import BaseProxy
 import numpy as np
 import poppy
 
@@ -9,10 +10,11 @@
 import catkit.hardware.iris_ao.util
 from catkit.interfaces.Instrument import SimInstrument
 import catkit.util
+from catkit.multiprocessing import MutexedNamespace
 from packaging.version import Version
 
 
-class PoppyIrisAODM(poppy.dms.HexSegmentedDeformableMirror):
+class PoppyIrisAODM(MutexedNamespace, poppy.dms.HexSegmentedDeformableMirror):
 
     @property
     def number_of_segments(self):
@@ -94,7 +96,7 @@ def __init__(self, config_id, dm, driver_serial):
 
         self.driver_serial = driver_serial
 
-        assert isinstance(dm, PoppyIrisAODM)
+        assert isinstance(dm, (PoppyIrisAODM, BaseProxy)), type(dm)
         self.dm = dm  # An instance of PoppyIrisAODM.
 
     def Popen(self,

catkit/emulators/newport/NewportMotorController.py

@@ -6,7 +6,7 @@ class NewportMotorControllerEmulator:
 
     def __init__(self):
         self.current_position = {}
-        self.log = logging.getLogger(__name__)
+        self.log = logging.getLogger()
 
     def XPS(self, *args, **kwargs):
         return self

catkit/emulators/npoint_tiptilt.py

@@ -30,7 +30,7 @@ def __init__(self):
         can read values, this is where we'll initialize some value stores that
         will get sent in emulated messages. """
 
-        self.log = logging.getLogger(f"{self.__module__}.{self.__class__.__qualname__}")
+        self.log = logging.getLogger()
         self.initialize()
 
     def initialize(self):

catkit/emulators/tests/test_emulated_boston_dm.py

@@ -77,21 +77,21 @@ def test_with(self):
         with self.instantiate_dm_controller() as dm:
             dm.apply_shape_to_both(flat_dm1, flat_dm2)
 
-        assert dm.instrument is None
+        assert not dm.is_open()
 
     def test_subsequent_with(self):
         flat_dm1 = DmCommand(np.zeros(self.number_of_actuators), 1)
         flat_dm2 = DmCommand(np.zeros(self.number_of_actuators), 2)
         with self.instantiate_dm_controller() as dm:
             dm.apply_shape_to_both(flat_dm1, flat_dm2)
 
-        assert dm.instrument is None
+        assert not dm.is_open()
 
         assert dm
         with dm:
             dm.apply_shape_to_both(flat_dm1, flat_dm2)
 
-        assert dm.instrument is None
+        assert not dm.is_open()
 
     def test_access_after_with(self):
         flat_dm1 = DmCommand(np.zeros(self.number_of_actuators), 1)
@@ -117,7 +117,7 @@ def test_keep_alive(self):
 
         dm.apply_shape_to_both(flat_dm1, flat_dm2)
         dm._Instrument__close()
-        assert dm.instrument is None
+        assert not dm.is_open()
         assert not dm._Instrument__keep_alive
 
     def test_del(self):

catkit/hardware/SnmpUps.py

@@ -8,7 +8,7 @@
 
 class SnmpUps(BackupPower):
 
-    log = logging.getLogger(__name__)
+    log = logging.getLogger()
 
     def __init__(self, config_id, ip, snmp_oid, pass_status, port=161, community="public"):
         self.config_id = config_id

catkit/hardware/WebPowerSwitch.py

@@ -19,7 +19,7 @@ class WebPowerSwitch(RemotePowerSwitch):
     instrument_lib = requests
 
     def initialize(self, user=None, password=None, ip=None, outlet_list={}):
-        self.log = logging.getLogger(__name__)
+        self.log = logging.getLogger()
 
         # Given the specificity of the script numbering I'm not sure that it really makes sense
         # to pass in these values, but hey.