From ee33287dc8f6137b5782385376e0e6bc2405dd90 Mon Sep 17 00:00:00 2001 From: philipstarkey Date: Fri, 26 Jun 2020 19:05:45 +1000 Subject: [PATCH 1/7] Relocate labscript_devices import machinery The labscript_devices import machinery has now been relocated to labscript_utils as the first step in breaking the circular dependency between labscript-devices and blacs packages. The code in _import_machinery.py is equivalent to that in labscript_devices/__init__.py exept that `__name__` has been replaced with `'labscript_devices'`. There is also some code that chooses whether to use the old labscript_devices code or the new labscript_utils code depending on the version of labscript_devices. The specific version will need to be adjusted to match whatever version removes the code from labscript_devices (and even then, would not catch the case of users with local changes in a development install) so this may not be the best approach, but I couldn't think of another way to ensure compatibility between old and new labscript-suite components without reintroducing a different circular dependency. The labscript_devices `__init__.py` file can then be replaced with `from labscript_utils.labscript_devices import *` so as not to need to update every device class file. --- labscript_utils/labscript_devices/__init__.py | 13 + .../labscript_devices/_import_machinery.py | 254 ++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 labscript_utils/labscript_devices/__init__.py create mode 100644 labscript_utils/labscript_devices/_import_machinery.py diff --git a/labscript_utils/labscript_devices/__init__.py b/labscript_utils/labscript_devices/__init__.py new file mode 100644 index 0000000..5eebdfe --- /dev/null +++ b/labscript_utils/labscript_devices/__init__.py @@ -0,0 +1,13 @@ +import importlib +import packaging.version + +labscript_device_version_str = importlib.metadata.version('labscript_devices') +labscript_devices_version = packaging.version.parse(labscript_device_version_str) + +# import and use the labscript_devices code if an old version of the package is used. +# This ensures that new labscript_utils with old labscript_devices doesn't attempt to +# register labscript_devices classes twice +if labscript_devices_version < packaging.version.parse('3.1.0.dev16'): + from labscript_devices import * +else: + from ._import_machinery import * \ No newline at end of file diff --git a/labscript_utils/labscript_devices/_import_machinery.py b/labscript_utils/labscript_devices/_import_machinery.py new file mode 100644 index 0000000..8d05534 --- /dev/null +++ b/labscript_utils/labscript_devices/_import_machinery.py @@ -0,0 +1,254 @@ +import os +import sys +import importlib +import imp +import warnings +import traceback +import inspect +from labscript_utils import dedent +from labscript_utils.labconfig import LabConfig + +"""This file contains the machinery for registering and looking up what BLACS tab and +runviewer parser classes belong to a particular labscript device. "labscript device" +here means a device that BLACS needs to communicate with. These devices have +instructions saved within the 'devices' group of the HDF5 file, and have a tab +corresponding to them in the BLACS interface. These device classes must have unique +names, such as "PineBlaster" or "PulseBlaster" etc. +There are two methods we use to find out which BLACS tab and runviewer parser correspond +to a device class: the "old" method, and the "new" method. The old method requires that +the the BLACS tab and runviewer parser be in a file called .py at the top +level of labscript_devices folder, and that they have class decorators @BLACS_tab or +@runviewer_parser to identify them. This method precludes putting code in subfolders or +splitting it across multiple files. +The "new" method is more flexible. It allows BLACS tabs and runviewer parsers to be +defined in any importable file within a subfolder of labscript_devices. Additionally, +the 'user_devices' configuration setting in labconfig can be used to specify a +comma-delimited list of names of importable packages containing additional labscript +devices. +Classes using the new method can be in files with any name, and do not need class +decorators. Instead, the classes should be registered by creating a file called +'register_classes.py', which when imported, makes calls to +labscript_devices.register_classes() to register which BLACS tab and runviewer parser +class belong to each device. Tab and parser classes must be passed to register_classes() +as fully qualified names, i.e. "labscript_devices.submodule.ClassName", not by passing +in the classes themselves. This ensures imports can be deferred until the classes are +actually needed. When BLACS and runviewer look up classes with get_BLACS_tab() and +get_runviewer_parser(), populate_registry() will be called in order to find all files +called 'register_classes.py' within subfolders (at any depth) of labscript_devices, and +they will be imported to run their code and hence register their classes. +The "new" method does not impose any restrictions on code organisation within subfolders +of labscript_devices, and so is preferable as it allows auxiliary utilities or resource +files to live in subfolders alongside the device code to which they are relevant, the +use of subrepositories, the grouping of similar devices within subfolders, and other +nice things to have. +The old method may be deprecated in the future. +""" + + +def _get_import_paths(import_names): + """For the given list of packages, return all folders containing their submodules. + If the packages do not exist, ignore them.""" + paths = [] + for name in import_names: + spec = importlib.util.find_spec(name) + if spec is not None and spec.submodule_search_locations is not None: + paths.extend(spec.submodule_search_locations) + return paths + + +def _get_device_dirs(): + """Return the directory of labscript_devices, and the folders containing + submodules of any packages listed in the user_devices labconfig setting""" + try: + user_devices = LabConfig().get('DEFAULT', 'user_devices') + except (LabConfig.NoOptionError, LabConfig.NoSectionError): + user_devices = 'user_devices' + # Split on commas, remove whitespace: + user_devices = [s.strip() for s in user_devices.split(',')] + return _get_import_paths(['labscript_devices'] + user_devices) + + +LABSCRIPT_DEVICES_DIRS = _get_device_dirs() + + +class ClassRegister(object): + """A register for looking up classes by module name. Provides a + decorator and a method for looking up classes decorated with it, + importing as necessary.""" + def __init__(self, instancename): + self.registered_classes = {} + # The name given to the instance in this namespace, so we can use it in error messages: + self.instancename = instancename + + def __call__(self, cls): + """Adds the class to the register so that it can be looked up later + by module name""" + # Add an attribute to the class so it knows its own name in case + # it needs to look up other classes in the same module: + cls.labscript_device_class_name = cls.__module__.split('.')[-1] + if cls.labscript_device_class_name == '__main__': + # User is running the module as __main__. Use the filename instead: + import __main__ + try: + cls.labscript_device_class_name = os.path.splitext(os.path.basename(__main__.__file__))[0] + except AttributeError: + # Maybe they're running interactively? Or some other funky environment. Either way, we can't proceed. + raise RuntimeError('Can\'t figure out what the file or module this class is being defined in. ' + + 'If you are testing, please test from a more standard environment, such as ' + + 'executing a script from the command line, or if you are using an interactive session, ' + + 'writing your code in a separate module and importing it.') + + # Add it to the register: + self.registered_classes[cls.labscript_device_class_name] = cls + return cls + + def __getitem__(self, name): + try: + # Ensure the module's code has run (this does not re-import it if it is already in sys.modules) + importlib.import_module('.' + name, 'labscript_devices') + except ImportError: + msg = """No %s registered for a device named %s. Ensure that there is a file + 'register_classes.py' with a call to + labscript_devices.register_classes() for this device, with the device + name passed to register_classes() matching the name of the device class. + Fallback method of looking for and importing a module in + labscript_devices with the same name as the device also failed. If using + this method, check that the module exists, has the same name as the + device class, and can be imported with no errors. Import error + was:\n\n""" + msg = dedent(msg) % (self.instancename, name) + traceback.format_exc() + raise ImportError(msg) + # Class definitions in that module have executed now, check to see if class is in our register: + try: + return self.registered_classes[name] + except KeyError: + # No? No such class is defined then, or maybe the user forgot to decorate it. + raise ValueError('No class decorated as a %s found in module %s, '%(self.instancename, 'labscript_devices' + '.' + name) + + 'Did you forget to decorate the class definition with @%s?'%(self.instancename)) + + +# Decorating labscript device classes and BLACS worker classes was never used for +# anything and has been deprecated. These decorators can be removed with no ill +# effects. Do nothing, and emit a warning telling the user this. +def deprecated_decorator(name): + def null_decorator(cls): + msg = '@%s decorator is unnecessary and can be removed' % name + warnings.warn(msg, stacklevel=2) + return cls + + return null_decorator + + +labscript_device = deprecated_decorator('labscript_device') +BLACS_worker = deprecated_decorator('BLACS_worker') + + +# These decorators can still be used, but their use will be deprecated in the future +# once all devices in mainline are moved into subfolders with a register_classes.py that +# will play the same role. For the moment we support both mechanisms of registering +# which BLACS tab and runviewer parser class belong to a particular device. +BLACS_tab = ClassRegister('BLACS_tab') +runviewer_parser = ClassRegister('runviewer_parser') + + +def import_class_by_fullname(fullname): + """Import and return a class defined by its fully qualified name as an absolute + import path, i.e. "module.submodule.ClassName".""" + split = fullname.split('.') + module_name = '.'.join(split[:-1]) + class_name = split[-1] + module = importlib.import_module(module_name) + return getattr(module, class_name) + + +def deprecated_import_alias(fullname): + """A way of allowing a class to be imported from an old location whilst a) not + actually importing it until it is instantiated and b) emitting a warning pointing to + the new import location. fullname must be a fully qualified class name with an + absolute import path. Use by calling in the module where the class used to be: + ClassName = deprecated_import_alias("new.path.to.ClassName")""" + calling_module_name = inspect.getmodule(inspect.stack()[1][0]).__name__ + cls = [] + def wrapper(*args, **kwargs): + if not cls: + cls.append(import_class_by_fullname(fullname)) + shortname = fullname.split('.')[-1] + newmodule = '.'.join(fullname.split('.')[:-1]) + msg = """Importing %s from %s is deprecated, please instead import it from + %s. Importing anyway for backward compatibility, but this may cause some + unexpected behaviour.""" + msg = dedent(msg) % (shortname, calling_module_name, newmodule) + warnings.warn(msg, stacklevel=2) + return cls[0](*args, **kwargs) + return wrapper + + +# Dictionaries containing the import paths to BLACS tab and runviewer parser classes, +# not the classes themselves. These will be populated by calls to register_classes from +# code within register_classes.py files within subfolders of labscript_devices. +BLACS_tab_registry = {} +runviewer_parser_registry = {} +# The script files that registered each device, for use in error messages: +_register_classes_script_files = {} + +# Wrapper functions to get devices out of the class registries. +def get_BLACS_tab(name): + if not BLACS_tab_registry: + populate_registry() + if name in BLACS_tab_registry: + return import_class_by_fullname(BLACS_tab_registry[name]) + # Fall back on file naming convention + decorator method: + return BLACS_tab[name] + + +def get_runviewer_parser(name): + if not runviewer_parser_registry: + populate_registry() + if name in runviewer_parser_registry: + return import_class_by_fullname(runviewer_parser_registry[name]) + # Fall back on file naming convention + decorator method: + return runviewer_parser[name] + + +def register_classes(labscript_device_name, BLACS_tab=None, runviewer_parser=None): + """Register the name of the BLACS tab and/or runviewer parser that belong to a + particular labscript device. labscript_device_name should be a string of just the + device name, i.e. "DeviceName". BLACS_tab_fullname and runviewer_parser_fullname + should be strings containing the fully qualified import paths for the BLACS tab and + runviewer parser classes, such as "labscript_devices.DeviceName.DeviceTab" and + "labscript_devices.DeviceName.DeviceParser". These need not be in the same module as + the device class as in this example, but should be within labscript_devices. This + function should be called from a file called "register_classes.py" within a + subfolder of labscript_devices. When BLACS or runviewer start up, they will call + populate_registry(), which will find and run all such files to populate the class + registries prior to looking up the classes they need""" + if labscript_device_name in _register_classes_script_files: + other_script =_register_classes_script_files[labscript_device_name] + msg = """A device named %s has already been registered by the script %s. + Labscript devices must have unique names.""" + raise ValueError(dedent(msg) % (labscript_device_name, other_script)) + BLACS_tab_registry[labscript_device_name] = BLACS_tab + runviewer_parser_registry[labscript_device_name] = runviewer_parser + script_filename = os.path.abspath(inspect.stack()[1][0].f_code.co_filename) + _register_classes_script_files[labscript_device_name] = script_filename + + +def populate_registry(): + """Walk the labscript_devices folder looking for files called register_classes.py, + and run them (i.e. import them). These files are expected to make calls to + register_classes() to inform us of what BLACS tabs and runviewer classes correspond + to their labscript device classes.""" + # We import the register_classes modules as a direct submodule of labscript_devices. + # But they cannot all have the same name, so we import them as + # labscript_devices._register_classes_script_ with increasing number. + module_num = 0 + for devices_dir in LABSCRIPT_DEVICES_DIRS: + for folder, _, filenames in os.walk(devices_dir): + if 'register_classes.py' in filenames: + # The module name is the path to the file, relative to the labscript suite + # install directory: + # Open the file using the import machinery, and import it as module_name. + fp, pathname, desc = imp.find_module('register_classes', [folder]) + module_name = 'labscript_devices._register_classes_%d' % module_num + _ = imp.load_module(module_name, fp, pathname, desc) + module_num += 1 From 5d651cb5110fbfe8802e7c2fabc732f909bea459 Mon Sep 17 00:00:00 2001 From: Phil Starkey Date: Sat, 27 Jun 2020 13:59:25 +1000 Subject: [PATCH 2/7] Fix importlib.metadata import on Python <3.8 Co-authored-by: Russell Anderson <5637107+rpanderson@users.noreply.github.com> --- labscript_utils/labscript_devices/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/labscript_utils/labscript_devices/__init__.py b/labscript_utils/labscript_devices/__init__.py index 5eebdfe..5b00910 100644 --- a/labscript_utils/labscript_devices/__init__.py +++ b/labscript_utils/labscript_devices/__init__.py @@ -1,4 +1,7 @@ -import importlib +try: + import importlib.metadata as importlib_metadata +except ImportError: + import importlib_metadata import packaging.version labscript_device_version_str = importlib.metadata.version('labscript_devices') @@ -10,4 +13,4 @@ if labscript_devices_version < packaging.version.parse('3.1.0.dev16'): from labscript_devices import * else: - from ._import_machinery import * \ No newline at end of file + from ._import_machinery import * From fb6892d9e680373449cafccc948798ac731d6c96 Mon Sep 17 00:00:00 2001 From: Phil Starkey Date: Sat, 27 Jun 2020 14:00:18 +1000 Subject: [PATCH 3/7] Fix importlib.metadata import on Python <3.8 Co-authored-by: Russell Anderson <5637107+rpanderson@users.noreply.github.com> --- labscript_utils/labscript_devices/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/labscript_devices/__init__.py b/labscript_utils/labscript_devices/__init__.py index 5b00910..8c7f5a2 100644 --- a/labscript_utils/labscript_devices/__init__.py +++ b/labscript_utils/labscript_devices/__init__.py @@ -4,7 +4,7 @@ import importlib_metadata import packaging.version -labscript_device_version_str = importlib.metadata.version('labscript_devices') +labscript_device_version_str = importlib_metadata.version('labscript_devices') labscript_devices_version = packaging.version.parse(labscript_device_version_str) # import and use the labscript_devices code if an old version of the package is used. From 6e0208ad3d26d84cb4f7ca3d19873ec696923094 Mon Sep 17 00:00:00 2001 From: philipstarkey Date: Sat, 27 Jun 2020 17:24:56 +1000 Subject: [PATCH 4/7] Handle case where labscript_devices is not installed --- labscript_utils/labscript_devices/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/labscript_utils/labscript_devices/__init__.py b/labscript_utils/labscript_devices/__init__.py index 8c7f5a2..20fcbfc 100644 --- a/labscript_utils/labscript_devices/__init__.py +++ b/labscript_utils/labscript_devices/__init__.py @@ -4,13 +4,18 @@ import importlib_metadata import packaging.version -labscript_device_version_str = importlib_metadata.version('labscript_devices') -labscript_devices_version = packaging.version.parse(labscript_device_version_str) +use_local_machinery = True +try: + labscript_device_version_str = importlib_metadata.version('labscript_devices') + labscript_devices_version = packaging.version.parse(labscript_device_version_str) + use_local_machinery = labscript_devices_version >= packaging.version.parse('3.1.0.dev16') +except importlib_metadata.PackageNotFoundError: + pass # import and use the labscript_devices code if an old version of the package is used. # This ensures that new labscript_utils with old labscript_devices doesn't attempt to # register labscript_devices classes twice -if labscript_devices_version < packaging.version.parse('3.1.0.dev16'): - from labscript_devices import * +if not use_local_machinery: + from labscript_devices import * else: from ._import_machinery import * From b09d8ad4a9028ffb158be9ed4fa73204e81a7066 Mon Sep 17 00:00:00 2001 From: philipstarkey Date: Sat, 27 Jun 2020 17:25:37 +1000 Subject: [PATCH 5/7] Rename files/folders to device_registry --- .../{labscript_devices => device_registry}/__init__.py | 0 .../_import_machinery.py => device_registry/_device_registry.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename labscript_utils/{labscript_devices => device_registry}/__init__.py (100%) rename labscript_utils/{labscript_devices/_import_machinery.py => device_registry/_device_registry.py} (100%) diff --git a/labscript_utils/labscript_devices/__init__.py b/labscript_utils/device_registry/__init__.py similarity index 100% rename from labscript_utils/labscript_devices/__init__.py rename to labscript_utils/device_registry/__init__.py diff --git a/labscript_utils/labscript_devices/_import_machinery.py b/labscript_utils/device_registry/_device_registry.py similarity index 100% rename from labscript_utils/labscript_devices/_import_machinery.py rename to labscript_utils/device_registry/_device_registry.py From c87ba99a98d5d4c797e3864697b294041d24b57b Mon Sep 17 00:00:00 2001 From: philipstarkey Date: Sat, 27 Jun 2020 17:30:06 +1000 Subject: [PATCH 6/7] Fixed import after folder rename --- labscript_utils/device_registry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/device_registry/__init__.py b/labscript_utils/device_registry/__init__.py index 20fcbfc..04e5441 100644 --- a/labscript_utils/device_registry/__init__.py +++ b/labscript_utils/device_registry/__init__.py @@ -18,4 +18,4 @@ if not use_local_machinery: from labscript_devices import * else: - from ._import_machinery import * + from ._device_registry import * From 39bef78f494be11bde697f60800c9abbc6f6ede4 Mon Sep 17 00:00:00 2001 From: chrisjbillington Date: Sat, 27 Jun 2020 14:35:53 -0400 Subject: [PATCH 7/7] Determine location of module by looking at module attributes Instead of version checking. Work around the circularity of importing new labscript_devices (which imports labscript_utils.device_registry) by unconditionally importing the new module first, rolling it back if it turns out labscript_devices is old and defines the device registry itself. --- labscript_utils/device_registry/__init__.py | 30 ++++++++----------- .../device_registry/_device_registry.py | 15 +++++++++- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/labscript_utils/device_registry/__init__.py b/labscript_utils/device_registry/__init__.py index 04e5441..93fb7f8 100644 --- a/labscript_utils/device_registry/__init__.py +++ b/labscript_utils/device_registry/__init__.py @@ -1,21 +1,15 @@ -try: - import importlib.metadata as importlib_metadata -except ImportError: - import importlib_metadata -import packaging.version +from ._device_registry import * -use_local_machinery = True +# Backwards compatibility for labscript-devices < 3.1. If labscript_devices defines the +# device registry as well, undo the above import and use the contents of +# labscript_devices instead. The above import must be done first so that the names are +# available to labscript_devices during the below import, since as of 3.1 it imports +# this module as well. try: - labscript_device_version_str = importlib_metadata.version('labscript_devices') - labscript_devices_version = packaging.version.parse(labscript_device_version_str) - use_local_machinery = labscript_devices_version >= packaging.version.parse('3.1.0.dev16') -except importlib_metadata.PackageNotFoundError: + from labscript_devices import ClassRegister + if ClassRegister.__module__ == 'labscript_devices': + for name in _device_registry.__all__: + del globals()[name] + from labscript_devices import * +except ImportError: pass - -# import and use the labscript_devices code if an old version of the package is used. -# This ensures that new labscript_utils with old labscript_devices doesn't attempt to -# register labscript_devices classes twice -if not use_local_machinery: - from labscript_devices import * -else: - from ._device_registry import * diff --git a/labscript_utils/device_registry/_device_registry.py b/labscript_utils/device_registry/_device_registry.py index 8d05534..256d76d 100644 --- a/labscript_utils/device_registry/_device_registry.py +++ b/labscript_utils/device_registry/_device_registry.py @@ -1,5 +1,4 @@ import os -import sys import importlib import imp import warnings @@ -45,6 +44,20 @@ class belong to each device. Tab and parser classes must be passed to register_c """ +__all__ = [ + 'LABSCRIPT_DEVICES_DIRS', + 'labscript_device', + 'BLACS_worker', + 'BLACS_tab', + 'runviewer_parser', + 'import_class_by_fullname', + 'deprecated_import_alias', + 'get_BLACS_tab', + 'get_runviewer_parser', + 'register_classes', +] + + def _get_import_paths(import_names): """For the given list of packages, return all folders containing their submodules. If the packages do not exist, ignore them."""