From ca2b988d7461860b79b09b65f3c4878462199371 Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Fri, 26 May 2017 14:10:04 +0100 Subject: [PATCH] Refactor and fix Sum calculation in VDSWrapperPart * Update to implement sum calculation * Fix some logging statements * Change sleeps to task.sleeps * Uncomment stat plugin in FemDetectorManager.yml --- .../blocks/excalibur/FemDetectorManager.yaml | 12 ++-- malcolm/imalcolm.py | 2 +- malcolm/parts/excalibur/vdswrapperpart.py | 60 ++++++++++--------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/malcolm/blocks/excalibur/FemDetectorManager.yaml b/malcolm/blocks/excalibur/FemDetectorManager.yaml index 4868e0d40..8af2409fa 100644 --- a/malcolm/blocks/excalibur/FemDetectorManager.yaml +++ b/malcolm/blocks/excalibur/FemDetectorManager.yaml @@ -22,13 +22,13 @@ name: FEM mri: $(mriPrefix):FEM -#- blocks.ADCore.StatsPlugin: -# mri: $(mriPrefix):STAT -# prefix: $(pvPrefix):STAT +- blocks.ADCore.StatsPlugin: + mri: $(mriPrefix):STAT + prefix: $(pvPrefix):STAT -#- parts.ADCore.StatsPluginPart: -# name: STAT -# mri: $(mriPrefix):STAT +- parts.ADCore.StatsPluginPart: + name: STAT + mri: $(mriPrefix):STAT - includes.ADCore.filewriting_collection: pvPrefix: $(pvPrefix) diff --git a/malcolm/imalcolm.py b/malcolm/imalcolm.py index 95214b9f4..096371028 100755 --- a/malcolm/imalcolm.py +++ b/malcolm/imalcolm.py @@ -136,7 +136,7 @@ def main(): from pkg_resources import require sys.path.insert(0, - "/home/mef65357/Detectors/VDS/vds-gen/venv/lib/python2.7/" + "/dls_sw/work/tools/RHEL6-x86_64/odin/venv/lib/python2.7/" "site-packages") require("tornado", "numpy", "cothread", "ruamel.yaml", "scanpointgenerator", "h5py") diff --git a/malcolm/parts/excalibur/vdswrapperpart.py b/malcolm/parts/excalibur/vdswrapperpart.py index 88abb3bed..7c839ba86 100644 --- a/malcolm/parts/excalibur/vdswrapperpart.py +++ b/malcolm/parts/excalibur/vdswrapperpart.py @@ -1,9 +1,8 @@ import os import sys -from time import sleep from subprocess import check_call -sys.path.insert(0, "/home/mef65357/Detectors/VDS/vds-gen/venv/lib/python2.7/" +sys.path.insert(0, "/dls_sw/work/tools/RHEL6-x86_64/odin/venv/lib/python2.7/" "site-packages") import h5py as h5 @@ -20,8 +19,8 @@ class VDSWrapperPart(Part): # Constants for vds-gen CLI app - VENV = "/home/mef65357/Detectors/VDS/vds-gen/venv/bin/python" - VDS_GEN = "/home/mef65357/Detectors/VDS/vds-gen/vdsgen/app.py" + VENV = "/dls_sw/work/tools/RHEL6-x86_64/odin/venv/bin/python" + VDS_GEN = "/dls_sw/work/tools/RHEL6-x86_64/odin/vds-gen/vdsgen/app.py" EMPTY = "-e" OUTPUT = "-o" FILES = "-f" @@ -132,11 +131,12 @@ def configure(self, task, completed_steps, steps_to_do, part_info, params): self.vds[node] = h5.ExternalLink(raw_file_path, node) # Create placeholder id and sum datasets - initial_shape = params.generator.shape + (1, 1) - max_shape = params.generator.shape + (None, 1) - self.vds.create_dataset(self.ID, initial_shape, maxshape=max_shape, - dtype="int32") - self.vds.create_dataset(self.SUM, initial_shape, dtype="float64") + initial_shape = (1, 1, 1, 1) + max_shape = params.generator.shape + (1, 1) + self.vds.create_dataset(self.ID, initial_shape, + maxshape=max_shape, dtype="int32") + self.vds.create_dataset(self.SUM, initial_shape, + maxshape=max_shape, dtype="float64") self._logger.debug("Calling vds-gen to create dataset in VDS") files = [self.RAW_FILE_TEMPLATE.format(fem) for fem in self.fems] @@ -157,7 +157,7 @@ def configure(self, task, completed_steps, steps_to_do, part_info, params): # Define output file path command += [self.OUTPUT, self.OUTPUT_FILE] command += [self.LOG_LEVEL, "1"] # str(self._logger.level / 10)] - self.log_warning("Command" + str(command)) + self.log_info("Command: %s", command) check_call(command) # Store required attributes @@ -182,35 +182,33 @@ def run(self, task, update_completed_steps): try: # Wait until raw files exist and have UniqueIDArray for path_ in self.raw_paths: - self._logger.info("Waiting for file %s to be created", path_) + self.log_info("Waiting for file %s to be created", path_) while not os.path.exists(path_): - sleep(1) + task.sleep(1) self.raw_datasets.append( h5.File(path_, self.READ, libver="latest", swmr=True)) for dataset in self.raw_datasets: - self._logger.info("Waiting for id in file %s", dataset) + self.log_info("Waiting for id in file %s", dataset) while self.ID not in dataset: - sleep(1) + task.sleep(1) - self._logger.info("Monitoring raw files until ID reaches %s", - self.done_when_reaches) + self.log_info("Monitoring raw files until ID reaches %s", + self.done_when_reaches) while self.id < self.done_when_reaches: + task.sleep(0.1) # Allow while loop to be aborted by controller ids = [] for dataset in self.raw_datasets: ids.append(self.get_id(dataset)) if min(ids) > self.id: - self._logger.info("Raw ID changed: %s - " - "Updating VDS ID and Sum", min(ids)) + self.log_info("Raw ID changed: %s - " + "Updating VDS ID and Sum", min(ids)) idx = ids.index(min(ids)) self.update_id(idx) - try: - self.update_sum() - except Exception: - self.log_warning("Can't access sum") - - self._logger.info("ID reached: " + str(self.id)) - except Exception: - self.log_error("Error in run") + self.update_sum() + + self.log_info("ID reached: %s", self.id) + except Exception as error: + self.log_error("Error in run. Message:\n%s", error.message) raise finally: self.close_files() @@ -224,19 +222,23 @@ def get_id(self, file_): file_[self.ID].refresh() return max(file_[self.ID].value.flatten()) else: - self.logger_.warning("File %s does not exist or does not have a " - "UniqueIDArray, returning 0", file_) + self.log_warning("File %s does not exist or does not have a " + "UniqueIDArray, returning 0", file_) return 0 def update_id(self, idx): min_id = self.raw_datasets[idx][self.ID] - self.vds[self.ID].resize(min_id.shape) + self.log_debug("ID shape:\n%s", min_id.shape) + self.vds[self.ID].resize(min_id.shape) self.vds[self.ID][...] = min_id def update_sum(self): sum_ = 0 for dataset in self.raw_datasets: + dataset[self.SUM].refresh() sum_ += dataset[self.SUM].value + self.log_debug("Sum shape:\n%s", sum_.shape) + self.vds[self.SUM].resize(sum_.shape) self.vds[self.SUM][...] = sum_