diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15cd4638..65dc50d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,6 @@ defaults: env: PACKAGE_NAME: blacs - SCM_VERSION_SCHEME: release-branch-semver SCM_LOCAL_SCHEME: no-local-version ANACONDA_USER: labscript-suite diff --git a/.gitignore b/.gitignore index 70c27dc9..583228d0 100644 --- a/.gitignore +++ b/.gitignore @@ -158,4 +158,5 @@ conda_packages/ # Sphinx documentation docs/html/ docs/source/_build/ -docs/source/components.rst \ No newline at end of file +docs/source/components.rst +docs/source/api/_autosummary \ No newline at end of file diff --git a/blacs/__main__.py b/blacs/__main__.py index 35668dcb..90984433 100644 --- a/blacs/__main__.py +++ b/blacs/__main__.py @@ -10,6 +10,8 @@ # the project for the full license. # # # ##################################################################### +'''BLACS GUI and supporting code +''' import labscript_utils.excepthook import os @@ -95,7 +97,7 @@ # Queue Manager Code from blacs.experiment_queue import QueueManager, QueueTreeview # Module containing hardware compatibility: -import labscript_devices +from labscript_utils import device_registry # Save/restore frontpanel code from blacs.front_panel_settings import FrontPanelSettings # Notifications system @@ -266,7 +268,7 @@ def __init__(self,application): self.settings_dict[device_name]["saved_data"] = tab_data[device_name]['data'] if device_name in tab_data else {} # Instantiate the device logger.info('instantiating %s'%device_name) - TabClass = labscript_devices.get_BLACS_tab(labscript_device_class_name) + TabClass = device_registry.get_BLACS_tab(labscript_device_class_name) self.tablist[device_name] = TabClass(self.tab_widgets[0],self.settings_dict[device_name]) except Exception: self.failed_device_settings[device_name] = {"front_panel": self.settings_dict[device_name]["front_panel_settings"], "save_data": self.settings_dict[device_name]["saved_data"]} @@ -602,7 +604,9 @@ def on_save_exit(self): self.front_panel_settings.save_front_panel_to_h5(self.settings_path,data[0],data[1],data[2],data[3],{"overwrite":True},force_new_conn_table=True) logger.info('Shutting down workers') for tab in self.tablist.values(): - tab.shutdown_workers() + # Tell tab to shutdown its workers if it has a method to do so. + if hasattr(tab, 'shutdown_workers'): + tab.shutdown_workers() QTimer.singleShot(100, self.finalise_quit) @@ -616,6 +620,14 @@ def finalise_quit(self, deadline=None, pending_threads=None): overdue = time.time() > deadline # Check for worker shutdown completion: for name, tab in list(self.tablist.items()): + # Immediately close tabs that don't support finalise_close_tab() + if not hasattr(tab, 'finalise_close_tab'): + try: + current_page = tab.close_tab(finalise=False) + except Exception as e: + logger.error('Couldn\'t close tab:\n%s' % str(e)) + del self.tablist[name] + continue fatal_error = tab.state == 'fatal error' if not tab.shutdown_workers_complete and overdue or fatal_error: # Give up on cleanly shutting down this tab's worker processes: diff --git a/blacs/__version__.py b/blacs/__version__.py index 699955f3..a88075ae 100644 --- a/blacs/__version__.py +++ b/blacs/__version__.py @@ -5,15 +5,14 @@ except ImportError: import importlib_metadata -VERSION_SCHEME = { - "version_scheme": os.getenv("SCM_VERSION_SCHEME", "guess-next-dev"), - "local_scheme": os.getenv("SCM_LOCAL_SCHEME", "node-and-date"), -} - root = Path(__file__).parent.parent if (root / '.git').is_dir(): from setuptools_scm import get_version - __version__ = get_version(root, **VERSION_SCHEME) + __version__ = get_version( + root, + version_scheme="release-branch-semver", + local_scheme=os.getenv("SCM_LOCAL_SCHEME", "node-and-date"), + ) else: try: __version__ = importlib_metadata.version(__package__) diff --git a/blacs/experiment_queue.py b/blacs/experiment_queue.py index 1844cf11..f98875e3 100644 --- a/blacs/experiment_queue.py +++ b/blacs/experiment_queue.py @@ -598,7 +598,7 @@ def restart_function(device_name): ) devices_in_use[name] = self.BLACS.tablist[name] start_order[name] = device_properties.get('start_order', None) - stop_order[name] = device_properties.get('start_order', None) + stop_order[name] = device_properties.get('stop_order', None) # Sort the devices into groups based on their start_order and stop_order start_groups = defaultdict(set) diff --git a/blacs/output_classes.py b/blacs/output_classes.py index 155880d0..1bf5febd 100644 --- a/blacs/output_classes.py +++ b/blacs/output_classes.py @@ -62,7 +62,7 @@ def __init__(self, hardware_name, connection_name, device_name, program_function if cls is None or not isinstance(calib_params, dict) or cls.base_unit != default_units: # log an error: reason = '' - if calib_class is None: + if cls is None: reason = f'The unit conversion class {calib_class} could not be imported. Ensure it is available on the computer running BLACS.' elif not isinstance(calib_params, dict): reason = 'The parameters for the unit conversion class are not a dictionary. Check your connection table code for errors and recompile it' diff --git a/docs/source/_templates/autosummary-class.rst b/docs/source/_templates/autosummary-class.rst new file mode 100644 index 00000000..4a2889ec --- /dev/null +++ b/docs/source/_templates/autosummary-class.rst @@ -0,0 +1,32 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :undoc-members: + :show-inheritance: + + {% block methods %} + .. automethod:: __init__ + + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + {% for item in methods %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/source/_templates/autosummary-module.rst b/docs/source/_templates/autosummary-module.rst new file mode 100644 index 00000000..c17f7334 --- /dev/null +++ b/docs/source/_templates/autosummary-module.rst @@ -0,0 +1,66 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: autosummary-class.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :template: autosummary-module.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst new file mode 100644 index 00000000..542c5f3a --- /dev/null +++ b/docs/source/api/index.rst @@ -0,0 +1,18 @@ +API Reference +============= + +.. autosummary:: + :toctree: _autosummary + :template: autosummary-module.rst + :recursive: + + blacs.analysis_submission + blacs.compile_and_restart + blacs.device_base_class + blacs.experiment_queue + blacs.front_panel_settings + blacs.notifications + blacs.output_classes + blacs.plugins + blacs.tab_base_classes + blacs.__main__ diff --git a/docs/source/conf.py b/docs/source/conf.py index 1bbadc91..5708f4ff 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -10,6 +10,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import copy import os from pathlib import Path from m2r import MdInclude @@ -39,6 +40,7 @@ # ones. extensions = [ "sphinx.ext.autodoc", + "sphinx.ext.autosummary", "sphinx.ext.autosectionlabel", "sphinx.ext.intersphinx", "sphinx.ext.napoleon", @@ -49,6 +51,26 @@ ] autodoc_typehints = 'description' +autosummary_generate = True +numfig = True +autodoc_mock_imports = ['labscript_utils'] + +# mock missing site package methods +import site +mock_site_methods = { + # Format: + # method name: return value + 'getusersitepackages': '', + 'getsitepackages': [] +} +__fn = None +for __name, __rval in mock_site_methods.items(): + if not hasattr(site, __name): + __fn = lambda *args, __rval=copy.deepcopy(__rval), **kwargs: __rval + setattr(site, __name, __fn) +del __name +del __rval +del __fn # Prefix each autosectionlabel with the name of the document it is in and a colon autosectionlabel_prefix_document = True @@ -223,3 +245,43 @@ def setup(app): img_path=img_path ) ) + + # hooks to test docstring coverage + app.connect('autodoc-process-docstring', doc_coverage) + app.connect('build-finished', doc_report) + + +members_to_watch = ['module', 'class', 'function', 'exception', 'method', 'attribute'] +doc_count = 0 +undoc_count = 0 +undoc_objects = [] +undoc_print_objects = False + + +def doc_coverage(app, what, name, obj, options, lines): + global doc_count + global undoc_count + global undoc_objects + + if (what in members_to_watch and len(lines) == 0): + # blank docstring detected + undoc_count += 1 + undoc_objects.append(name) + else: + doc_count += 1 + + +def doc_report(app, exception): + global doc_count + global undoc_count + global undoc_objects + # print out report of documentation coverage + total_docs = undoc_count + doc_count + if total_docs != 0: + print(f'\nAPI Doc coverage of {doc_count/total_docs:.1%}') + if undoc_print_objects or os.environ.get('READTHEDOCS'): + print('\nItems lacking documentation') + print('===========================') + print(*undoc_objects, sep='\n') + else: + print('No docs counted, run \'make clean\' then rebuild to get the count.') \ No newline at end of file diff --git a/docs/source/device-tabs.rst b/docs/source/device-tabs.rst new file mode 100644 index 00000000..a9c35806 --- /dev/null +++ b/docs/source/device-tabs.rst @@ -0,0 +1,315 @@ +Device Tabs +=========== + +BLACS creates a tab, in the GUI, for each device it is to control. This information is +sourced from a lab connection table, defined using the labscript API, which is kept up to +date with the current configuration of hardware in the lab. Much of the BLACS GUI is thus +dynamically generated, creating an interface suited to a particular apparatus configuration +rather than enforcing a particular style. These tabs encapsulate three components: the code +that produces the graphical interface, the worker process(es) (which communicate with the +actual hardware), and a state machine which handles communication between the GUI and +the worker process(es). + +The graphical interface +----------------------- + +Each tab GUI is generated from a set of standard components in order to bring uniformity +to the control of heterogeneous hardware. This also simplifies the process of adding support +for new hardware devices (see :doc:`labscript-devices:adding_devices`) as the author of the device code does not require +knowledge of the GUI widget toolkit. Each tab comprises the following sections (see :numref:`fig-PulseBlaster-Tab` +and :numref:`fig-PCIe6363-Tab`): + +#. device management shortcuts (such as restarting the device), +#. a region (usually hidden) for displaying error messages from the worker process, +#. arrays of ‘manual’ controls for interacting with each of the device’s input and output + channels when shots are not running, +#. custom controls specific to a particular device (for example status indicators), and +#. the current state of the state machine (see :ref:`device-tabs:State machine`). + +.. _fig-PulseBlaster-Tab: + +.. figure:: img/blacs_pulseblastertab.png + + An example of a BLACS tab for a PulseBlaster DDS-II-300-AWG device. The numbered labels + match the listing in :ref:`device-tabs:The graphical interface`. + +.. _fig-PCIe6363-Tab: + +.. figure:: img/blacs_pcie6363tab.png + + An example of a BLACS tab for an NI PCIe-6363 device. The numbered labels match the + listing in :ref:`device-tabs:The graphical interface`. + +The most prominent feature is the arrays of manual controls. These are particularly +useful for manual debugging of an experiment apparatus outside of running shots. For +easy identification, each channel is automatically named with both the hardware output +port, and any assigned name from the lab connection table. All analog values also have +an associated dropdown list, where the current unit is displayed. Unit conversions are +automatically determined from the lab connection table (where they are defined using the +labscript API). This makes debugging simpler as you can immediately be sure of +the output quantity in real world units (for example, the strength of a magnetic field). All +output controls can be locked via a right-click context menu to prevent accidental change +of their state, which is particularly important when controlling sensitive equipment that +can be damaged. For analog quantities, the default step size used when incrementing or +decrementing the value [1]_ can also be customised via the right-click context menu. + +The values displayed in the manual controls are also coupled to the hardware device +capabilities. The device code that programs the hardware (see worker processes in :ref:`device-tabs:Worker processes`) +has the ability to return a new value for each channel, each time the device is programmed, +allowing the quantised, rounded or coerced value to be returned such that the manual control +faithfully displays the output state. BLACS also provides an architecture to periodically +poll device values for devices that support such queries. This is particularly important for +devices that are not physically restricted to being controlled by a single user (for example, +devices controlled via a web interface) or devices that don’t remember their state after +being power cycled. For such devices, BLACS continually compares the device state with +the values displayed in the GUI. If a difference is detected, BLACS presents the user with +options to select either the device state or the local state on a per output basis (see +:numref:`fig-monitoring`). + +.. _fig-monitoring: + +.. figure:: img/blacs_monitoring.png + + An example showing how devices in BLACS can monitor the consistency between + the front panel state and the output of the device (when not running a shot). Here we show a + Novatech DDS9m device that has just been power cycled, which causes the output states to + reset to a default setting. BLACS detects an inconsistency between the front panel values of + BLACS and the output state reported by the device, and presents the GUI pictured above. + The user can then to choose either use the local or remote value for each output channel. + Once selected, the front panel values of BLACS are updated to the selected value and the + device is reprogrammed to match, restoring consistency. + +Worker processes +---------------- + +For each device, BLACS launches at least one separate Python process (a worker process) +for communicating with the hardware. BLACS communicates with the worker process +through our own remote procedure call (RPC) protocol. The python process(es) run a +thin wrapper around a specified Python class, which allows the parent process (in this case +BLACS) to remotely call methods of the class in the worker process. A method in the +worker process is invoked by the tab state machine (see :ref:`device-tabs:State machine`), via a message sent over a +ZMQ socket. The only task of a worker process is to process any data that is sent to it (via +the invocation of one of its methods), interact appropriately with the hardware device(s) it +manages, and return any relevant data to the state machine. A third party software library, +used to interact with a hardware device (typically provided by a hardware manufacturer), +is then only loaded within the isolated worker process. There are several benefits to this +‘sandboxing’ model. Details on writing the +code for a worker process can be found in :doc:`labscript-devices:adding_devices`. + +As previously implied, we have implemented the ability for a BLACS device tab to spawn +multiple worker processes. This is particularly useful for devices that handle both inputs +and output, and whose API allows these inputs and outputs to be separated and managed +by separate processes. An example of such a device is a National Instruments acquisition +card such as the NI PCIe-6363. For this device, we spawn three worker processes: the first +handles analog and digital outputs, the second handles analog acquisition and the third +handles monitoring of a counter in order to measure the lengths of any waits. +Multiprocessing also results in a reduction in device programming time prior to the +start of an experiment shot. Most device programming is I/O bound (not limited by the +processing power of the PC). Simultaneously programming all devices used in a shot thus +typically completes in the time it takes to program the longest device (rather than the sum +of all programming times for sequential programming). + +State machine +------------- + +One of the major changes in BLACS v2.0 (written and released after our paper [2]_ was +published) was the introduction of a more advanced state machine for each device tab. State +machines are an important tool in building complex systems as they enforce a workflow (in +this case, for GUI-hardware interaction) which improves the stability of the control system. +By using a state machine, we enforce control over what actions can be taken at any given +time, improving the robustness of our control software. For example, manual controls on the +BLACS front panel should not be able to control hardware devices that are under precision +timing while executing a shot. A state machine allows such events to either be discarded or +queued until an appropriate time, under a consistent set of easily defined rules. + +The aim of this state machine is to manage the execution of the device-specific code +described previously, which falls into the categories of GUI code and worker-process code. +This code exists within Python methods of the device classes, +and so will be referred to in this section as the execution of a ‘GUI method’ and a ‘worker +process method’ respectively. We have implemented a non-standard nested state machine, +for which we will coin the term 2D state machine. It consists of two orthogonal sets of +states (which we term the inner and outer states) which are linked by the device code. This +architecture differs from a standard nested state machine in that it is not hierarchical (events +are not passed to the parent state machine as in the hierarchical finite state machine). Our +implementation is also unique in that the workflow of the inner (dimension of the) state +machine is identical regardless of the outer state. + +The outer dimension follows a classical state machine. There are four possible states +(which we call modes to distinguish them from the states on the inner dimension): + +* mode_manual, +* mode_transition_to_buffered, +* mode_buffered, and +* mode_transition_to_manual. + +These four modes represent the two possible modes of operation for the hardware; manual +control from BLACS or stepping through instructions during execution of an experiment +shot, and the transitions between these modes (where the programming required to change +the mode of the device, for example the programming of hardware instructions, usually +takes place). + +The inner dimension of this two-dimensional state machine is similar to the state machine +that existed in BLACS v1.0. There are 5 possible states: + +* idle, +* execute (part of) GUI method, +* request execution of worker process method via ZMQ, +* wait for results from worker process method via ZMQ, and +* fatal error. + +The inner state machine spends the majority of time in the idle state where it waits for an +event to become available from a queue. Events are typically placed in the queue in response +to user actions (for example clicking one of the manual control buttons), the ‘queue manager’ +processing a shot (see :doc:`shot-management`), or the timeout of a timer (for example for regular status checks +of the hardware). + +We define GUI methods that may be queued in the inner state by using a Python +decorator, which abstracts away the state machine so that users can call the Python method +as normal and be assured that it will always run as part of the state machine (although this +enforces the requirement that such methods will return immediately and not return a result +to the caller). The decorator itself takes arguments that indicate the modes (of the outer +state machine) that the GUI method is allowed to run in, and whether the method should +remain in the event queue until the outer state machine enters a mode where it can run, or +if it should just be deleted once it reaches the head of the queue. We also provide an option +to only run the most recent entry for a method if duplicate entries for the GUI method +exist in the queue (albeit with different arguments). This is particularly useful for methods +that take a long time to complete but which may be queued up rapidly, for instance a user +rapidly changing output values of a device that is slow to program. An example of how you +might use the state machine is shown in the definition of a GUI method below. + +.. code-block:: python + + class MyDevice(blacs.device_base_classes.DeviceTab): + # only run in MODE_MANUAL and keep the state in the queue until + # the mode condition is met + @define_state(MODE_MANUAL, True) + def transition_to_buffered(self, h5_file, notify_queue): + # set the mode to MODE_TRANSITION_TO_BUFFERED + self.mode = MODE_TRANSITION_TO_BUFFERED + # define the set of arguments and keyword arguments + # to be passed to the worker processes + args, kwargs = (h5_file,) , {} + # Yield to the state machine so that the worker process + # can be run + result = yield(self.queue_work(self.primary_worker, + 'transition_to_buffered', *args, **kwargs)) + # check that everything was successful ... + if result : + # success ! + # update the mode and notify the caller + self.mode = MODE_BUFFERED + notify_queue.put([self.device_name, 'success']) + else: + # Failure ! + # notify the caller + notify_queue.put([self.device_name, 'fail']) + # queue up a method in the state machine + # to return to MODE_MANUAL and instruct the + # worker to program the device ready for + # manual operation + self.abort_transition_to_buffered() + + @define_state(MODE_TRANSITION_TO_BUFFERED, False ) + def abort_transition_to_buffered(self): + +This is an example of how one might define the GUI method for triggering the programming +of devices so that they are ready for buffered execution of a shot (ready to step +through hardware instructions). The GUI method transition_to_buffered has been decorated +in order to ensure it is only run as part of the state machine, which means the method +will sit in the inner state machine’s event queue until the outer state machine mode is set +to ‘MODE_MANUAL’. When finally executed by the state machine, the method updates +the mode of the outer state machine, and yields to the inner state machine in order to tell a +worker process to transition into buffered mode (which typically involves programming the +table of hardware instructions from the hdf5 shot file). If successful, the outer state machine +mode is updated again and the caller of the method (in this case the ‘Queue Manager’) is +notified of the result. If unsuccessful, we call the abort_transition_to_buffered method +(which is also decorated as a GUI method) which queues up a new event for the inner state +machine. In practice, common functionality like these methods are abstracted away from +the user and contained within the blacs.device_base_classes.DeviceTab class. They +are implemented in a similar (but more generalised) way to the code shown here. For example +transition_to_buffered is actually written to support an arbitrary number of worker +processes. Further information on adding support for new devices (and how to use the state +machine architecture) is included in :doc:`labscript-devices:adding_devices`. + +The state machine for each device tab runs in its own thread and follows a well defined +workflow (also shown graphically in figure 6.4) which can be influenced by the device code. +When an event is available in the queue (that can run in the current mode of the outer state +machine), the inner state machine transitions to the ‘execute GUI method’ state, and calls +the Python method that was queued up. As this method likely interacts with the GUI, the +method is executed in the main thread (via a request to the GUI event loop provided by +the Qt widget toolkit). This method executes (temporarily blocking interaction with the +GUI) until it either completes, or hits the yield Python keyword. The yield keyword in +Python returns control of the program to the calling code (in this case our state machine). +The yield keyword also allows the called method to return information to the calling code +(for example the data variable would be returned if called as yield(data)). We utilise this +to allow the GUI method to request that the inner state machine transition through the +inner states relating to the worker process, in this case by calling: + +.. code-block:: python + + yield(self.queue_work('worker_name', 'worker_method_name')) + +If such a statement is encountered, the inner state machine enters the ‘request execution of +worker method’ state and requests the named worker process execute the named method. It +then immediately transitions to the ‘wait for results from worker’ state. Once results have +been received from the worker process (after it has run the worker method), the inner state +machine re-enters the ‘execute GUI method’ state, passing in the results from the worker +process as the return value of the yield keyword, and continues with the execution of the +GUI method from the point it left. This continues until the execution of the GUI method +is complete, where the state machine then enters the ‘idle’ state again. The exception to +this is if a Python Exception is raised in the GUI method, in which case the state machine +enters the ‘Fatal error’ state. The GUI method may also request a change in the outer state +machine mode, which then determines which events can be processed when the inner state +machine next returns to the ‘Idle’ state. + +.. _fig-statemachine: + +.. figure:: img/blacs_statemachine.png + + A flowchart of the logic for the BLACS state machine as described in :ref:`device-tabs:State machine`. + +This results in a flexible state machine architecture that allows the device code to control +some portions of the state machine, while maintaining a fixed state machine structure across +device tabs. By exposing the internals of the state machine only via the BLACS tab GUI +methods, we can abstract away much of the state machine implementation and simplify +the necessary coding skills needed to implement support for new devices. We believe this +is a critical requirement of meeting the flexibility goal of our control system, and further +details on the simplicity of adding support for new devices is discussed in :doc:`labscript-devices:adding_devices`. Our state +machine architecture also allows us to provide a consistent and responsive GUI to a user +by obscuring hardware specific details and offloading these to a separate process. + +Handling waits +-------------- + +In order to use waits, one of the devices in BLACS must monitor +the length of each wait so that a mapping between labscript time and experiment time can +be made. This mapping is then used by analog acquisition devices in BLACS to correctly +break up acquired traces into the requested acquisitions. The length of the wait is also used +in real (software) time by the wait monitor in order to ensure the experiment is not stuck +in a wait forever. + +The current wait monitor implementation uses one of the inbuilt counters that are in +many National Instruments acquisition devices, however other implementations are of course +possible if support is added when creating the device classes. At the completion +of an experiment shot, the wait monitor calculates the durations of each wait (based on +data it acquired during the shot) and writes these to the shot file. The wait monitor then +broadcasts a ZMQ event indicating this has been completed. Device code that relies on the +wait information (for example, for breaking up analog acquisitions into the requested set +of traces), waits for this event to be received during the transition to manual mode, before +performing any action. This ensures that the measured lengths of the waits are always +available in the hdf5 file when required. + + +.. rubric:: Footnotes + +.. [1] Incrementing or decrementing the value can be done using the up/down arrows next to the value, the + mouse scroll wheel, or the arrow keys on the keyboard. The page up/down keys can also be used, which + will adjust the value by 10 times the step size. This is distinct from typing a value directly into the + widget, which is not affected by the step size. However both incrementing/decrementing and typing + a value in will be equally affected by any quantisation demanded by the hardware device, which we + discuss in the following paragraph. + +.. [2] P. T. Starkey, C. J. Billington, S. P. Johnstone, M. Jasperse, K. Helmerson, + L. D. Turner, and R. P. Anderson. *A scripted control system for autonomous + hardware-timed experiments* Review of Scientific Instruments **84**, 085111 (2013). + https://doi.org/10.1063/1.4817213 diff --git a/docs/source/img/blacs_monitoring.png b/docs/source/img/blacs_monitoring.png new file mode 100644 index 00000000..1e71dda7 Binary files /dev/null and b/docs/source/img/blacs_monitoring.png differ diff --git a/docs/source/img/blacs_overview.png b/docs/source/img/blacs_overview.png new file mode 100644 index 00000000..5547cfb6 Binary files /dev/null and b/docs/source/img/blacs_overview.png differ diff --git a/docs/source/img/blacs_pcie6363tab.png b/docs/source/img/blacs_pcie6363tab.png new file mode 100644 index 00000000..eac188e7 Binary files /dev/null and b/docs/source/img/blacs_pcie6363tab.png differ diff --git a/docs/source/img/blacs_pulseblastertab.png b/docs/source/img/blacs_pulseblastertab.png new file mode 100644 index 00000000..58dc3d66 Binary files /dev/null and b/docs/source/img/blacs_pulseblastertab.png differ diff --git a/docs/source/img/blacs_queueflowchart.png b/docs/source/img/blacs_queueflowchart.png new file mode 100644 index 00000000..eafffcc1 Binary files /dev/null and b/docs/source/img/blacs_queueflowchart.png differ diff --git a/docs/source/img/blacs_queuemanagement.png b/docs/source/img/blacs_queuemanagement.png new file mode 100644 index 00000000..01f81af0 Binary files /dev/null and b/docs/source/img/blacs_queuemanagement.png differ diff --git a/docs/source/img/blacs_statemachine.png b/docs/source/img/blacs_statemachine.png new file mode 100644 index 00000000..517b174c Binary files /dev/null and b/docs/source/img/blacs_statemachine.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index 2993f86d..d1e4a413 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,11 +6,17 @@ blacs ===== +A graphical interface to scientific instruments and experiment supervision. + .. toctree:: :maxdepth: 2 :hidden: :caption: DOCUMENTATION + introduction + usage + api/index + .. toctree:: :maxdepth: 2 :hidden: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst new file mode 100644 index 00000000..f17d898b --- /dev/null +++ b/docs/source/introduction.rst @@ -0,0 +1,42 @@ +Introduction +============ + +BLACS interfaces with the hardware devices controlling an experiment, and manages the +queue of shots to execute on the apparatus. BLACS has two modes of operation: the +execution of shots under hardware timing, and the manual control of hardware devices (by +the user) via the BLACS GUI. The interface is shown in :numref:`fig-overview` and is split into two +sections that align with the two modes of operation: the queue of shots to execute, and a +GUI interface for manually controlling the output state of the hardware devices when not +running shots (which can be useful for manual debugging of an apparatus). + +.. _fig-overview: + +.. figure:: img/blacs_overview.png + + The BLACS interface. Left: The shot queue. Main: A set of tabs (one for each hardware device) that + provide a manual control interface + for each device. Further details on the BLACS interface can be found in :doc:`usage`. + +The shot queue contains standard controls for adding deleting and reordering shots. The +queue can also be paused or put into one of several modes that repeat the shots in the queue. +When a shot finishes, and the results have been saved to the hdf5 file, the shot may be +optionally sent to the lyse server specified in the GUI. + +The GUI for each hardware device is dynamically generated at runtime, based on a +connection table written using the labscript API. A device tab is created for each device, and +communicates with a unique worker process which, in turn, handles the communication with +the hardware device. The device tab GUI is populated with controls for each input/output +(IO) channel present. To aid in the identification of a relevant I/O channel, controls are +labelled in BLACS using both the hardware I/O port name and a user specified name from +the connection table of a labscript file. Analog channels (or more complex output types +like a DDS that are represented by several analog numbers) also integrate with the unit +conversions specified in the connection table, allowing both control and the display of units +other than the (device specific) default hardware unit. Channels connected to sensitive +equipment can have the output values limited or the control locked entirely to prevent +accidental changes. Output values are stored on exit, and restored on start-up, to avoid +unexpected output transients. + +.. rubric:: Footnotes + +.. [1] Documentation taken from Phillip T. Starkey *A software framework for control and automation of precisely timed experiments* + Thesis, Monash University (2019) https://doi.org/10.26180/5d1db8ffe29ef \ No newline at end of file diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst new file mode 100644 index 00000000..9e6456d4 --- /dev/null +++ b/docs/source/plugins.rst @@ -0,0 +1,43 @@ +Plugins +======= + +To cater to the variety of lab environments, BLACS supports custom user code (that is not +covered by a device implementation) in the form of plugins. Plugins allow custom graphical +interfaces to be created through the addition of menu items, notifications, preferences editable +through a common preferences panel, and custom tabs that sit alongside device tabs. +Plugins are also provided access to a variety of internal BLACS objects, such as the shot +queue, and can register callback functions to be run when certain events happen (such as a +shot completing). This provides a powerful basis for customising the behaviour of BLACS +in a way that is both modular and maintainable, providing a way to include optional conflicting +features without needing to resolve the incompatibility. Plugins can be easily shared +between groups, allowing for a diverse variety of control system interfaces that are all built +on a common platform. We have developed several plugins at Monash, which are detailed +in the following sections, and demonstrate the broad applicability of the plugin system. + +The API refrence for the standard plugins is :doc:`here` + +The connection table plugin +--------------------------- + +This plugin is included in the default install of BLACS, and provides a clean interface to +manage the lab connection table that BLACS uses to automatically generate the device +tabs and their graphical interfaces. The plugin inserts a menu item that provides shortcuts +for: + +#. editing the connection table Python file, +#. initiating the recompilation of the connection table, and +#. editing the preferences that control the behaviour of the plugin. + +The preferences panel allows you to configure a list of hdf5 files containing runmanager +globals to use during the compilation of the connection table (commonly used as unit +conversion parameters), as well as a list of unit conversion Python files, to watch for any +changes. At startup, the plugin launches a background thread that monitors changes to +these files, as well as the connection table Python file and compiled connection table hdf5 +file. If any modifications are detected, a notification is shown at the top of BLACS informing +that the lab connection table should be recompiled. If recompilation is chosen by the user, +the plugin manages the recompilation of the connection table using the runmanager API +and output from this process is displayed in a window. Once recompilation is successful, +the plugin relaunches BLACS so that the new connection table is loaded. This ensures that +BLACS is using the same knowledge of the experiment apparatus as any future shots will +when they are created by runmanager (assuming they share the globals files used). + diff --git a/docs/source/pyqt5-modified-objects.inv b/docs/source/pyqt5-modified-objects.inv new file mode 100644 index 00000000..418cb896 Binary files /dev/null and b/docs/source/pyqt5-modified-objects.inv differ diff --git a/docs/source/shot-management.rst b/docs/source/shot-management.rst new file mode 100644 index 00000000..366e95e1 --- /dev/null +++ b/docs/source/shot-management.rst @@ -0,0 +1,82 @@ +Shot Management +=============== + +The primary purpose of BLACS is to execute experiment shots on the lab apparatus. File +paths to shots are typically received by BLACS over ZMQ, but can also be loaded directly +through the BLACS GUI (either via the file menu, or by dragging and dropping onto the +queue). Prior to accepting the shot, BLACS compares the connection table of the shot to the +lab connection table and ensures that the shot is compatible with the current configuration of +the laboratory hardware. Connection table compatibility requires that the shot connection +table is a subset of the lab connection table. This ensures that old experiments can not be +run on hardware that is no-longer configured correctly, preventing damage or unexpected +results. Shots that pass this check are added to a queue, which is visible in the BLACS +GUI (see :numref:`fig-queue`). + +.. _fig-queue: + +.. figure:: img/blacs_queuemanagement.png + + The queue manager GUI within BLACS (with the device tabs hidden). (1) + The pause button stops the queue from processing new shots (a currently running shot + will finish). (2) The repeat button, when enabled, will duplicate a completed shot and + either place the duplicate at the bottom or the top of the queue (depending on the mode + selected). (3) The abort button immediately stops the execution of the current shot and + returns hardware to manual mode. (4-5) Buttons to add or delete selected shots from the + queue. (6) A button to clear the entire queue. (7-10) Buttons to reorder selected shots + within the queue. (11) The current status of the queue is displayed here. For example, the + status may indicate that devices are currently being programmed, the master pseudoclock + has been triggered and the experiment is running, or that acquired data is currently being + saved into the hdf5 shot file. (12) The list of shot files in the queue, in the order they + will be executed (the topmost is executed first). (13) A button to enable or disable the + forwarding of shots to lyse for analysis. (14) The network hostname of the PC running lyse. + +The queue is processed by a thread in BLACS, which we term the ‘queue manager’, that +takes the top-most shot in the queue and, in turn, executes it. Shot execution follows the +following pattern (a flowchart of this process is also shown in :numref:`fig-flowchart`): + +#. For each device in use in the shot, a message is sent to the corresponding device tab + state machine indicating that the device should program the device for hardware timed + execution of a shot. These messages are sent asynchronously, which ensures devices + program in parallel if possible (subject to the state machine being available to process + the message). Included in this message is the path to the hdf5 shot file, which each + device tab ultimately passes to a worker process that in turn, reads out the hardware + instructions and programs the hardware. During this programming, the device tab + enters the mode ‘transition_to_buffered’ (see §6.1.1.3). +#. The queue manager then waits until all devices have reported they have programmed, + at which point all device tabs in use should be in the ‘buffered’ state machine mode. + If a device does not report it has completed within a 5 minute timeout, or a device + reports an error has occurred during programming, the queue manager aborts the shot + by pausing the queue, instructing all device tabs to abort, and replacing the shot at + the top of the queue. +#. Provided all devices report they are ready, the queue manager proceeds with starting + the shot. This involves recording the current state of all manual controls (as these usually + affect the initial values of the shot and may affect results in certain experiments) + and then instructing the master pseudoclock to begin execution of the programmed + instructions. +#. The queue manager then waits for the master pseudoclock to report that the experiment + shot has completed. If an error occurs in a device tab during a shot, the queue + manager aborts the shot (as previously described) and pauses the queue. +#. Once a shot has completed, the queue manager instructs all device tabs to ‘transition + to manual’ mode. At this stage, device tabs enter the ‘transition_to_manual’ state + machine mode where they save any acquired data and reprogram the hardware device + for manual operation via the BLACS GUI. Again, if errors occur during this process, + the queue manager aborts the shot as before, but with the additional step of cleaning + any saved data from the hdf5 file (so that the shot file is returned to the state prior + to execution). +#. The path to the shot file is now sent to a separate thread that runs a routine for + managing submission of shots to lyse for analysis. This routine forwards the shot file + paths to the lyse server specified in the BLACS GUI if analysis submission is enabled + (see figure 6.5 (13–14)). If lyse does not respond to these messages, the shot file paths + are buffered until such time as lyse does respond, to ensure no shots are missing from + analysis. +#. Finally, the queue manager checks the state of the repeat button in the BLACS GUI + and, if required, duplicates the shot (minus the acquired data) and places the duplicate + in the appropriate place in the queue. + +.. _fig-flowchart: + +.. figure:: img/blacs_queueflowchart.png + + A flowchart of the logic for the BLACS queue manager. For brevity, we have + not included the logic for pausing the queue via the GUI or handling error conditions. See + the listing above for further details. \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 00000000..5ba184b1 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,26 @@ +Usage +===== + +BLACS is the primary interface between experiment shot files created by runmanager, and +the hardware devices that control the apparatus. BLACS provides a graphical interface for +users to manage the execution of shots, and manually control the output state of hardware +devices. In order to support heterogenous hardware, the functionality of BLACS can be +extended by developers (who implement support for custom devices) through the provided +BLACS API. BLACS thus broadly consists of a set of device code that interfaces with +the hardware and provides programmatic and manual control of that hardware, which is +discussed in :doc:`device-tabs`, and a shot management routine that receives shot files from runmanager +and schedules their execution on the apparatus, which is discussed in :doc:`shot-management`. + +BLACS is also readily extensible using a plugin system, discussed in :doc:`plugins`. + +.. toctree:: + :maxdepth: 2 + + device-tabs + shot-management + plugins + +.. rubric:: Footnotes + +.. [1] Documentation taken from Phillip T. Starkey *A software framework for control and automation of precisely timed experiments* + Thesis, Monash University (2019) https://doi.org/10.26180/5d1db8ffe29ef \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e6ca9c75..4cd0fd9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools", "wheel", "setuptools_scm"] +requires = ["setuptools", "wheel", "setuptools_scm>=4.1.0"] build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 815d189f..1bc3e6f1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ classifiers = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 [options] zip_safe = False @@ -26,9 +27,10 @@ packages = find: python_requires = >=3.6 install_requires = importlib_metadata - labscript_devices>=3.0.0 - labscript_utils>=3.0.0 + labscript_utils>=3.1.0b1 + runmanager>=3.0.0 qtutils>=2.3.2 + setuptools_scm>=4.1.0 zprocess>=2.14.1 [options.entry_points] @@ -41,7 +43,7 @@ gui_scripts = pyqt = PyQt5 docs = PyQt5 - Sphinx==3.0.1 - sphinx-rtd-theme==0.4.3 + Sphinx==3.5.3 + sphinx-rtd-theme==0.5.2 recommonmark==0.6.0 m2r==0.2.1 diff --git a/setup.py b/setup.py index 12b22ca2..f11f9907 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ import os from setuptools import setup -VERSION_SCHEME = { - "version_scheme": os.getenv("SCM_VERSION_SCHEME", "guess-next-dev"), - "local_scheme": os.getenv("SCM_LOCAL_SCHEME", "node-and-date"), -} - -setup(use_scm_version=VERSION_SCHEME) +setup( + use_scm_version={ + "version_scheme": "release-branch-semver", + "local_scheme": os.getenv("SCM_LOCAL_SCHEME", "node-and-date"), + } +)