Merge pull request #629 from mantidproject/11607_mask_poldi_spectra_a…

…utomatically

Mask detectors in POLDI data automatically

Code/Mantid/Framework/PythonInterface/plugins/algorithms/PoldiLoadRuns.py

@@ -9,6 +9,8 @@
 class PoldiLoadRuns(PythonAlgorithm):
     _nameTemplate = ""
     _mergeCheckEnabled = True
+    _autoMaskBadDetectors = True
+    _autoMaskThreshold = 3.0
 
     def category(self):
         return "SINQ\\Poldi"
@@ -45,6 +47,15 @@ def PyInit(self):
         self.declareProperty('EnableMergeCheck', True, direction=Direction.Input,
                              doc="Enable all the checks in PoldiMerge. Do not deactivate without very good reason.")
 
+        self.declareProperty('MaskBadDetectors', True, direction=Direction.Input,
+                             doc=('Automatically disable detectors with unusually small or large values, in addition'
+                                  ' to those masked in the instrument definition.'))
+
+        self.declareProperty('BadDetectorThreshold', 3.0, direction=Direction.Input,
+                             doc=('Detectors are masked based on how much their intensity (integrated over time) '
+                                  'deviates from the median calculated from all detectors. This parameter indicates '
+                                  'how many times bigger the intensity needs to be for a detector to be masked.'))
+
         self.declareProperty(WorkspaceProperty(name='OutputWorkspace',
                                                defaultValue='',
                                                direction=Direction.Output),
@@ -93,6 +104,10 @@ def PyExec(self):
         # PoldiMerge checks that instruments are compatible, but it can be disabled (calibration measurements)
         self._mergeCheckEnabled = self.getProperty('EnableMergeCheck').value
 
+        # The same for removing additional dead or misbehaving wires
+        self._autoMaskBadDetectors = self.getProperty('MaskBadDetectors').value
+        self._autoMaskThreshold = self.getProperty('BadDetectorThreshold').value
+
         # Get a list of output workspace names.
         outputWorkspaces = self.getLoadedWorkspaceNames(year, mergeRange, mergeWidth)
 
@@ -172,8 +187,13 @@ def getLoadedWorkspaceNames(self, year, mergeRange, mergeWidth):
                 for j in range(i, i + mergeWidth - 1):
                     DeleteWorkspace(self._nameTemplate + str(j))
 
-            # If the workspace is still valid (merging could have failed), it's appended to the output.
+            # If the workspace is still valid (merging could have failed), it's processed further
             if AnalysisDataService.doesExist(currentTotalWsName):
+                # If the option is enabled, mask detectors that are likely to be misbehaving
+                if self._autoMaskBadDetectors:
+                    self.log().information("Masking bad detectors automatically.")
+                    self.autoMaskBadDetectors(currentTotalWsName)
+
                 outputWorkspaces.append(currentTotalWsName)
 
         return outputWorkspaces
@@ -184,6 +204,19 @@ def loadAndTruncateData(self, workspaceName, year, j):
         LoadInstrument(workspaceName, InstrumentName="POLDI")
         PoldiTruncateData(InputWorkspace=workspaceName, OutputWorkspace=workspaceName)
 
+    # Automatically determine bad detectors and mask them
+    def autoMaskBadDetectors(self, currentTotalWsName):
+        Integration(currentTotalWsName, OutputWorkspace='integrated')
+
+        MedianDetectorTest('integrated', SignificanceTest=3.0, HighThreshold=self._autoMaskThreshold, HighOutlier=200, \
+                           CorrectForSolidAngle=False, OutputWorkspace='maskWorkspace')
+
+        MaskDetectors(Workspace=AnalysisDataService.retrieve(currentTotalWsName), MaskedWorkspace='maskWorkspace')
+
+        # Clean up
+        DeleteWorkspace('integrated')
+        DeleteWorkspace('maskWorkspace')
+
     # Returns true if the supplied workspace is a WorkspaceGroup
     def isGroupWorkspace(self, workspace):
         return issubclass(type(workspace), WorkspaceGroup)

Code/Mantid/Testing/SystemTests/tests/analysis/POLDILoadRunsTest.py

@@ -4,6 +4,7 @@
 from mantid.api import *
 import numpy as np
 
+
 class POLDILoadRunsTest(stresstesting.MantidStressTest):
     """This assembly of test cases checks that the behavior of PoldiLoadRuns is correct."""
 
@@ -19,6 +20,8 @@ def runTest(self):
         self.loadWorkspacesDontOverwriteOther()
         self.loadWorkspacesOverwriteOther()
 
+        self.checkRemoveBadDetectors()
+
     def loadSingleWorkspace(self):
         singleWs = PoldiLoadRuns(2013, 6904)
 
@@ -128,6 +131,36 @@ def loadWorkspacesDontOverwriteOther(self):
 
         self.assertTrue(issubclass(type(otherWs), Workspace))
 
+    def checkRemoveBadDetectors(self):
+        # Determine bad detectors automatically
+        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
+                                            BadDetectorThreshold=2.5)
+
+        wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
+        self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
+            x).isMasked()]), 36)
+
+        self.clearAnalysisDataService()
+
+        # Lower threshold, more excluded detectors
+        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
+                                            BadDetectorThreshold=2.0)
+
+        wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
+        self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
+            x).isMasked()]), 49)
+
+        self.clearAnalysisDataService()
+
+        # Only use those from the IDF
+        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=False)
+
+        wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
+        self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
+            x).isMasked()]), 12)
+
+        self.clearAnalysisDataService()
+
     def compareWorkspaces(self, left, right):
         for i in range(left.getNumberHistograms()):
             self.assertTrue(np.array_equal(left.dataY(i), right.dataY(i)))

Code/Mantid/docs/source/algorithms/PoldiFitPeaks2D-v1.rst

@@ -156,6 +156,6 @@ The refined lattice parameter is printed at the end:
 
 .. testoutput:: ExSilicon2DPawley
 
-    Refined lattice parameter a = 5.43126 +/- 5e-05
+    Refined lattice parameter a = 5.43125 +/- 4e-05
 
 .. categories::

Code/Mantid/docs/source/algorithms/PoldiLoadRuns-v1.rst

@@ -9,7 +9,11 @@
 Description
 -----------
 
-This algorithm makes it easier to load POLDI data. Besides importing the raw data (:ref:`algm-LoadSINQ`), it performs the otherwise manually performed steps of instrument loading (:ref:`algm-LoadInstrument`), truncation (:ref:`algm-PoldiTruncateData`). To make the algorithm more useful, it is possible to load data from multiple runs by specifying a range. In many cases, data files need to be merged in a systematic manner, which is also covered by this algorithm. For this purpose there is a parameter that specifies how the files of the specified range should be merged. The data files are named following the scheme `group_data_run`, where `group` is the name specified in `OutputWorkspace` and `run` is the run number and placed into a WorkspaceGroup with the name given in `OutputWorkspace`.
+This algorithm makes it easier to load POLDI data. Besides importing the raw data (:ref:`algm-LoadSINQ`), it performsthe otherwise manually performed steps of instrument loading (:ref:`algm-LoadInstrument`), truncation (:ref:`algm-PoldiTruncateData`). To make the algorithm more useful, it is possible to load data from multiple runs by specifying a range. In many cases, data files need to be merged in a systematic manner, which is also covered by this algorithm. For this purpose there is a parameter that specifies how the files of the specified range should be merged. The data files are named following the scheme `group_data_run`, where `group` is the name specified in `OutputWorkspace` and `run` is the run number and placed into a WorkspaceGroup with the name given in `OutputWorkspace`.
+
+By default, detectors that show unusually large integrated intensities are excluded if the pass a certain threshold
+with respect to the median of integrated intensities in all detectors. The threshold can be adjusted using an
+additional parameter. Detectors that are masked in the instrument definition are always masked.
 
 The data loaded in this way can be used directly for further processing with :ref:`algm-PoldiAutoCorrelation`.