Merge pull request #21 from percival-detector/dev

Improved logging, fixed bug in monitoring channels.

percival/detector/adapter.py

@@ -94,22 +94,35 @@ def put(self, path, request):  # pylint: disable=W0613
         logging.debug("%s", request.body)
 
         status_code = 200
-        response = {}
+        response = {'response': 'Failed',
+                    'error': 'Unknown',
+                    'command': 'Unknown',
+                    'param_names': 'Unknown',
+                    'parameters': 'Unknown',
+                    'time': 'Unknown'}
 
         # Create a new Percival Command object
         try:
             cmd = Command(request)
+            response['command'] = cmd.command_name
+            response['param_names'] = cmd.param_names
+            response['parameters'] = cmd.parameters
+            response['time'] = cmd.command_time
             if 'start' in cmd.command_name:
                 self._auto_read = True
             elif 'stop' in cmd.command_name:
                 self._auto_read = False
             else:
-                response = self._detector.execute_command(cmd)
+                #cmd_response = self._detector.execute_command(cmd)
+                self._detector.queue_command(cmd)
+                # Merge the response from the command execution
+                #for key in cmd_response:
+                #    response[key] = cmd_response[key]
         except Exception as ex:
             # Return an error condition with the exception message
-            status_code = 500
-            response['error'] = ex.args
-            response['trace'] = traceback.format_exc()
+            # status_code = 500
+            response['response'] = 'Failed'
+            response['error'] = "{} => {}".format(ex.args, traceback.format_exc())
 
         logging.debug(response)
 

percival/detector/command.py

@@ -54,6 +54,9 @@ def __init__(self, request):
         self._log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
         self._command_type = None
         self._command_name = None
+        self._command_active = False
+        self._command_state = 'Unknown'
+        self._command_message = ''
         self._parameters = {}
         self._trace = {
             CommandTrace.user: "unknown",
@@ -76,13 +79,50 @@ def command_name(self):
     def command_time(self):
         return self._trace[CommandTrace.creation_time]
 
+    @property
+    def active(self):
+        return self._command_active
+
+    @property
+    def state(self):
+        return self._command_state
+
+    @property
+    def message(self):
+        return self._command_message
+
+    @property
+    def param_names(self):
+        return list(self._parameters.keys())
+
+    @property
+    def parameters(self):
+        parsed_params = {}
+        for key in self._parameters:
+            param_value = str(self._parameters[key])
+            if len(param_value) > 40:
+                param_value = "{ too long to display }"
+            parsed_params[key] = param_value
+        return parsed_params
+
     @property
     def format_trace(self):
         return {'Username': self._trace[CommandTrace.user],
                 'Created': self._trace[CommandTrace.creation_time],
                 'Source_Address': self._trace[CommandTrace.origin_address],
                 'Source_ID': self._trace[CommandTrace.origin_type]}
 
+    def activate(self):
+        self._command_active = True
+        self._command_state = 'Active'
+
+    def complete(self, success, message=''):
+        if success:
+            self._command_state = 'Completed'
+        else:
+            self._command_state = 'Failed'
+        self._command_message = message
+
     def has_param(self, name):
         found_name = False
         if name in self._parameters:

percival/detector/detector.py

@@ -6,9 +6,16 @@
 from __future__ import print_function
 import os
 import logging
+import threading
 from percival.log import get_exclusive_file_logger
 from datetime import datetime, timedelta
 import getpass
+import sys
+is_py2 = sys.version[0] == '2'
+if is_py2:
+    import Queue as queue
+else:
+    import queue as queue
 
 from percival.carrier import const
 from percival.carrier.buffer import SensorBufferCommand
@@ -525,6 +532,7 @@ def __init__(self, ini_file=None, download_config=True, initialise_hardware=True
         self._sensor = None
         self._control_groups = None
         self._monitor_groups = None
+        self._active_command = None
         self._log.info("Creating SystemSettings object")
         self._system_settings = SystemSettings()
         self._log.info("Creating ChipRadoutSettings object")
@@ -551,8 +559,12 @@ def __init__(self, ini_file=None, download_config=True, initialise_hardware=True
         if initialise_hardware:
             self._log.info("Executing initialisation of channels")
             self.initialize_channels()
+        self._command_queue = queue.Queue()
+        self._command_thread = threading.Thread(target=self.command_loop)
+        self._command_thread.start()
 
     def cleanup(self):
+        self.queue_command(None)
         self._setpoint_control.stop_scan_loop()
 
     def load_ini(self):
@@ -581,6 +593,7 @@ def setup_control(self):
         self._board_settings[const.BoardTypes.bottom] = BoardSettings(self._txrx, const.BoardTypes.bottom)
         self._board_settings[const.BoardTypes.carrier] = BoardSettings(self._txrx, const.BoardTypes.carrier)
         self._board_settings[const.BoardTypes.plugin] = BoardSettings(self._txrx, const.BoardTypes.plugin)
+        self._board_values[const.BoardTypes.bottom] = BoardValues(self._txrx, const.BoardTypes.bottom)
         self._board_values[const.BoardTypes.carrier] = BoardValues(self._txrx, const.BoardTypes.carrier)
         self._system_settings.set_txrx(self._txrx)
         self._chip_readout_settings.set_txrx(self._txrx)
@@ -789,6 +802,23 @@ def load_setpoints(self, setpoint_ini):
         self._percival_params.load_setpoint_group_ini(setpoint_ini)
         self._setpoint_control.load_ini(self._percival_params.setpoint_params)
 
+    def queue_command(self, command):
+        self._command_queue.put(command)
+
+    def command_loop(self):
+        running = True
+        while running:
+            try:
+                command = self._command_queue.get()
+                if command:
+                    self.execute_command(command)
+                else:
+                    running = False
+            except PercivalDetectorError as e:
+                self._active_command.complete(success=False, message=str(e))
+            except Exception as e:
+                self._active_command.complete(success=False, message="Unhandled exception: {}".format(str(e)))
+
     def execute_command(self, command):
         """
         Log the execution of the command (use command trace)
@@ -797,51 +827,58 @@ def execute_command(self, command):
         :param command:
         :return:
         """
-        response = {'command': command.command_name,
-                    'time': command.command_time}
+        response = {}
         # Check if the command is a PUT command
         if 'PUT' in command.command_type:
             # Log the trace information from the command object
             self._trace_log.info("Command {} [{}] executed".format(command.command_type, command.command_name))
+            command.activate()
             self._trace_log.info(command.format_trace)
+            self._active_command = command
             # Check if the command is a connection request to the DB
             if command.command_name in str(PercivalCommandNames.cmd_download_channel_cfg):
                 # No parameters required for this command
                 self.load_configuration()
+                self._active_command.complete(success=True)
             if command.command_name in str(PercivalCommandNames.cmd_connect_db):
                 # No parameters required for this command
                 self.setup_db()
+                self._active_command.complete(success=True)
             elif command.command_name in str(PercivalCommandNames.cmd_load_config):
                 # Parameter [config_type] one of setpoint, ctrl_group, mon_group, channels
                 # Parameter [config] path to config file or the configuration contents
                 if command.has_param('config_type'):
                     if command.has_param('config'):
-                        config_type = command.get_param('config_type')
-                        config_desc = command.get_param('config').replace('::', '=')
-                        if 'setpoints' in config_type:
-                            self.load_setpoints(config_desc)
-                        elif 'control_groups' in config_type:
-                            self.load_control_groups(config_desc)
-                        elif 'monitor_groups' in config_type:
-                            self.load_monitor_groups(config_desc)
-                        elif 'system_settings' in config_type:
-                            self.load_system_settings(config_desc)
-                            self.download_system_settings()
-                        elif 'chip_readout_settings' in config_type:
-                            self.load_chip_readout_settings(config_desc)
-                            self.download_chip_readout_settings()
-                        elif 'clock_settings' in config_type:
-                            self.load_clock_settings(config_desc)
-                            self.download_clock_settings()
-                        elif 'sensor_configuration' in config_type:
-                            self.load_sensor_configuration(config_desc)
-                            self.download_sensor_configuration()
-                        elif 'sensor_calibration' in config_type:
-                            self.load_sensor_calibration(config_desc)
-                            self.download_sensor_calibration()
-                        elif 'sensor_debug' in config_type:
-                            self.load_sensor_debug(config_desc)
-                            self.download_sensor_debug()
+                        if len(command.get_param('config')) > 0:
+                            config_type = command.get_param('config_type')
+                            config_desc = command.get_param('config').replace('::', '=')
+                            if 'setpoints' in config_type:
+                                self.load_setpoints(config_desc)
+                            elif 'control_groups' in config_type:
+                                self.load_control_groups(config_desc)
+                            elif 'monitor_groups' in config_type:
+                                self.load_monitor_groups(config_desc)
+                            elif 'system_settings' in config_type:
+                                self.load_system_settings(config_desc)
+                                self.download_system_settings()
+                            elif 'chip_readout_settings' in config_type:
+                                self.load_chip_readout_settings(config_desc)
+                                self.download_chip_readout_settings()
+                            elif 'clock_settings' in config_type:
+                                self.load_clock_settings(config_desc)
+                                self.download_clock_settings()
+                            elif 'sensor_configuration' in config_type:
+                                self.load_sensor_configuration(config_desc)
+                                self.download_sensor_configuration()
+                            elif 'sensor_calibration' in config_type:
+                                self.load_sensor_calibration(config_desc)
+                                self.download_sensor_calibration()
+                            elif 'sensor_debug' in config_type:
+                                self.load_sensor_debug(config_desc)
+                                self.download_sensor_debug()
+                            self._active_command.complete(success=True)
+                        else:
+                            self._active_command.complete(success=False, message='Empty configuration parameter supplied')
                     else:
                         raise PercivalDetectorError("No config provided (file or object)")
                 else:
@@ -897,6 +934,7 @@ def execute_command(self, command):
                 if command.has_param('name'):
                     # Execute the system command
                     self.system_command(command.get_param('name'))
+                    self._active_command.complete(success=True)
 
         # Check if the command is a GET command
         elif 'GET' in command.command_type:
@@ -1010,6 +1048,24 @@ def read(self, parameter):
                      "hardware": self._txrx.get_status()
                      }
 
+        elif parameter == "action":
+            if self._active_command:
+                reply = {'response': self._active_command.state,
+                         'error': self._active_command.message,
+                         'command': self._active_command.command_name,
+                         'param_names': self._active_command.param_names,
+                         'parameters': self._active_command.parameters,
+                         'time': self._active_command.command_time
+                         }
+            else:
+                reply = {'response': '',
+                         'error': '',
+                         'command': '',
+                         'param_names': '',
+                         'parameters': '',
+                         'time': ''
+                         }
+
         elif parameter == "groups":
             # Construct dictionaries of control and monitor groups
             reply = {"control_groups": {"group_names": []},
@@ -1129,5 +1185,23 @@ def update_status(self):
                     if self._db:
                         self._db.log_point(time_now, name, self._monitors[name].status)
 
+            response = self._board_values[const.BoardTypes.bottom].read_values()
+            # time_now = datetime.today() - timedelta(hours=1)
+            time_now = datetime.utcnow()
+            self._log.debug(response)
+            read_maps = generate_register_maps(response)
+            self._log.debug(read_maps)
+
+            readback_block = BoardValueRegisters[const.BoardTypes.bottom]
+            for addr, value in response:  # pylint: disable=W0612
+                offset = addr - readback_block.start_address
+                name = self._percival_params.monitoring_channel_name_by_index_and_board_type(offset,
+                                                                                             const.BoardTypes.bottom)
+                if name in self._monitors:
+                    self._monitors[name].update(read_maps[offset])
+                    status_msg[name] = self._monitors[name].status
+                    if self._db:
+                        self._db.log_point(time_now, name, self._monitors[name].status)
+
             self._log.debug("Status: %s", status_msg)
         return status_msg

percival/mkdir_handler.py

@@ -15,17 +15,17 @@ def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None,
         dir = os.path.dirname(filename)
         if not os.path.exists(dir):
             os.makedirs(dir)
-        self._clean_directory(filename)
+        #self._clean_directory(filename)
         RotatingFileHandler.__init__(self, filename, mode, maxBytes, backupCount, encoding, delay)
 
-    def _clean_directory(self, filename):
-        file = os.path.basename(filename)
-        directory = os.path.dirname(filename)
-        for the_file in os.listdir(directory):
-            if file in the_file:
-                file_path = os.path.join(directory, the_file)
-                try:
-                    if os.path.isfile(file_path):
-                        os.unlink(file_path)
-                except Exception as e:
-                    print(e)
+    #def _clean_directory(self, filename):
+    #    file = os.path.basename(filename)
+    #    directory = os.path.dirname(filename)
+    #    for the_file in os.listdir(directory):
+    #        if file in the_file:
+    #            file_path = os.path.join(directory, the_file)
+    #            try:
+    #                if os.path.isfile(file_path):
+    #                    os.unlink(file_path)
+    #            except Exception as e:
+    #                print(e)