Merge pull request #654 from mantidproject/11632_update_poldi_dic_for…

…_2015

Enable loading of 2015 POLDI data

Code/Mantid/Framework/PythonInterface/plugins/algorithms/LoadSINQFile.py

@@ -13,6 +13,7 @@
 from mantid import config
 import os.path
 import numpy as np
+import re
 
 #--------- place to look for dictionary files
 
@@ -47,26 +48,32 @@ def PyExec(self):
             "HRPT":"hrpt.dic",
             "MARSI":"marsin.dic",
             "MARSE":"marse.dic",
+            "POLDI_legacy":"poldi_legacy.dic",
             "POLDI":"poldi.dic",
             "RITA-2":"rita.dic",
             "SANS":"sans.dic",
             "SANS2":"sans.dic",
             "TRICS":"trics.dic"\
         }
+
+        lookupInstrumentName = inst
+        if inst == 'POLDI':
+            lookupInstrumentName = self._getPoldiLookupName(fname, lookupInstrumentName)
+
         dictsearch = os.path.join(config['instrumentDefinition.directory'],"nexusdictionaries")
-        dicname = os.path.join(dictsearch, diclookup[inst])
+        dicname = os.path.join(dictsearch, diclookup[lookupInstrumentName])
         wname = "__tmp"
         ws = mantid.simpleapi.LoadFlexiNexus(fname,dicname,OutputWorkspace=wname)
 
         if inst == "POLDI":
             if ws.getNumberHistograms() == 800:
                 ws.maskDetectors(SpectraList=range(0,800)[::2])
 
-            config.appendDataSearchDir(config['groupingFiles.directory'])
-            grp_file = "POLDI_Grouping_800to400.xml"
-            ws = mantid.simpleapi.GroupDetectors(InputWorkspace=ws,
-                                                 OutputWorkspace=wname,
-                                                 MapFile=grp_file, Behaviour="Sum")
+                config.appendDataSearchDir(config['groupingFiles.directory'])
+                grp_file = "POLDI_Grouping_800to400.xml"
+                ws = mantid.simpleapi.GroupDetectors(InputWorkspace=ws,
+                                                     OutputWorkspace=wname,
+                                                     MapFile=grp_file, Behaviour="Sum")
 
             # Reverse direction of POLDI data so that low index corresponds to low 2theta.
             histogramCount = ws.getNumberHistograms()
@@ -86,5 +93,19 @@ def PyExec(self):
         # delete temporary reference
         mantid.simpleapi.DeleteWorkspace(wname,EnableLogging=False)
 
+    def _getPoldiLookupName(self, fname, lookupInstrumentName):
+        year = self._extractYearFromFileName(fname)
+        if year < 2015:
+            return lookupInstrumentName + '_legacy'
+        # Otherwise, this is the current POLDI format.
+        return lookupInstrumentName
+
+    def _extractYearFromFileName(self, filename):
+        pureFileName = os.path.basename(filename)
+        pattern = re.compile(r'\w+(\d{4})n[\w\d\.]+')
+        matches = re.match(pattern, pureFileName)
+
+        return int(matches.group(1))
+
 #---------- register with Mantid
 AlgorithmFactory.subscribe(LoadSINQFile)

Code/Mantid/Framework/PythonInterface/plugins/algorithms/PoldiMerge.py

@@ -69,6 +69,11 @@ def canMerge(self, leftWorkspace, rightWorkspace):
         if not self.timingsMatch(leftWorkspace.dataX(0), rightWorkspace.dataX(0)):
             raise RuntimeError("Timings don't match")
 
+        # If this option is enabled, don't do any checks
+        if not self.checkInstruments:
+            self.log().warning('Instrument check has been disabled.')
+            return True
+
         leftRun = leftWorkspace.getRun()
         rightRun = rightWorkspace.getRun()
 
@@ -99,7 +104,7 @@ def instrumentsMatch(self, leftWorkspace, rightWorkspace):
         leftInstrument = leftWorkspace.getInstrument()
         rightInstrument = rightWorkspace.getInstrument()
 
-        return (not self.checkInstruments) or self.instrumentParametersMatch(leftInstrument, rightInstrument)
+        return self.instrumentParametersMatch(leftInstrument, rightInstrument)
 
     def instrumentParametersMatch(self, leftInstrument, rightInstrument):
         if not leftInstrument.getDetector(0).getPos() == rightInstrument.getDetector(0).getPos():
@@ -119,8 +124,13 @@ def getParameterValue(self, instrument, parameterTuple):
 
     def propertiesMatch(self, leftRun, rightRun):
         for propertyName in self.comparedPropertyNames:
-            if abs(self.getPropertyValue(leftRun.getProperty(propertyName)) - self.getPropertyValue(rightRun.getProperty(propertyName))) > 5e-3:
-                raise RuntimeError("Property '%s' does not match" % (propertyName))
+            if leftRun.hasProperty(propertyName) and rightRun.hasProperty(propertyName):
+                leftProperty = leftRun.getProperty(propertyName)
+                rightProperty = rightRun.getProperty(propertyName)
+                if abs(self.getPropertyValue(leftProperty) - self.getPropertyValue(rightProperty)) > 5e-3:
+                    raise RuntimeError("Property '%s' does not match" % (propertyName))
+            else:
+                self.log().warning('Property ' + propertyName + ' is not present in data - skipping comparison.')
 
         return True
 

Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/PoldiMergeTest.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 
+
 class PoldiMergeTest(unittest.TestCase):
     def __init__(self, *args):
         unittest.TestCase.__init__(self, *args)
@@ -38,8 +39,9 @@ def __init__(self, *args):
 
             self.goodTimingBadProperties.getRun().addProperty(p, badProperty, True)
 
-    def __runMerge__(self, workspaceNames):
-        return PoldiMerge(WorkspaceNames=workspaceNames, OutputWorkspace="PoldiMergeOutput", CheckInstruments=False)
+    def __runMerge__(self, workspaceNames, checkInstruments=False):
+        return PoldiMerge(WorkspaceNames=workspaceNames, OutputWorkspace="PoldiMergeOutput",
+                          CheckInstruments=checkInstruments)
 
     def test_happyCase(self):
         output = self.__runMerge__("Base,GoodTiming")
@@ -84,12 +86,13 @@ def test_timingOffset(self):
         self.assertFalse(AnalysisDataService.doesExist("PoldiMergeOutput"))
 
     def test_badProperties(self):
-        self.assertRaises(RuntimeError, lambda: self.__runMerge__("Base,GoodTimingBadProperties"))
+        self.assertRaises(RuntimeError, lambda: self.__runMerge__("Base,GoodTimingBadProperties", True))
         self.assertFalse(AnalysisDataService.doesExist("PoldiMergeOutput"))
 
     def test_badName(self):
         self.assertRaises(RuntimeError, lambda: self.__runMerge__("Base,NotExisting"))
         self.assertFalse(AnalysisDataService.doesExist("PoldiMergeOutput"))
 
+
 if __name__ == '__main__':
     unittest.main()

Code/Mantid/Testing/Data/SystemTest/poldi2015n000977.hdf.md5

@@ -0,0 +1 @@
+ba219b53856c045827563542e4c40ad1

Code/Mantid/Testing/SystemTests/tests/analysis/LoadLotsOfFiles.py

@@ -68,7 +68,8 @@
                 'poldi2013n006903.hdf',
                 'poldi2013n006904.hdf',
                 'poldi2014n019874.hdf',
-                'poldi2014n019881.hdf'
+                'poldi2014n019881.hdf',
+                'poldi2015n000977.hdf'
                 ]
 
 EXPECTED_EXT = '.expected'

Code/Mantid/Testing/SystemTests/tests/analysis/POLDILoadRunsTest.py

@@ -21,6 +21,7 @@ def runTest(self):
         self.loadWorkspacesOverwriteOther()
 
         self.checkRemoveBadDetectors()
+        self.check2015PoldiData()
 
     def loadSingleWorkspace(self):
         singleWs = PoldiLoadRuns(2013, 6904)
@@ -133,8 +134,9 @@ def loadWorkspacesDontOverwriteOther(self):
 
     def checkRemoveBadDetectors(self):
         # Determine bad detectors automatically
-        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
-                                            BadDetectorThreshold=2.5)
+        PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
+                                            BadDetectorThreshold=2.5,
+                                            OutputWorkspace='twoWorkspacesMerged')
 
         wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
         self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
@@ -143,8 +145,9 @@ def checkRemoveBadDetectors(self):
         self.clearAnalysisDataService()
 
         # Lower threshold, more excluded detectors
-        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
-                                            BadDetectorThreshold=2.0)
+        PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=True,
+                                            BadDetectorThreshold=2.0,
+                                            OutputWorkspace='twoWorkspacesMerged')
 
         wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
         self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
@@ -153,14 +156,27 @@ def checkRemoveBadDetectors(self):
         self.clearAnalysisDataService()
 
         # Only use those from the IDF
-        twoWorkspacesMerged = PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=False)
+        PoldiLoadRuns(2013, 6903, 6904, 2, MaskBadDetectors=False,
+                                            OutputWorkspace='twoWorkspacesMerged')
 
         wsMerged = AnalysisDataService.retrieve("twoWorkspacesMerged_data_6904")
         self.assertEquals(len([True for x in range(wsMerged.getNumberHistograms()) if wsMerged.getDetector(
             x).isMasked()]), 12)
 
         self.clearAnalysisDataService()
 
+    def check2015PoldiData(self):
+        PoldiLoadRuns(2015, 977, OutputWorkspace='ws')
+
+        ws2015 = AnalysisDataService.retrieve('ws_data_977')
+        self.assertEquals(ws2015.getNumberHistograms(), 400)
+        self.assertEquals(len(ws2015.readX(0)), 125)
+        self.assertTrue(ws2015.run().hasProperty('chopperspeed'))
+
+        self.clearAnalysisDataService()
+
+
+
     def compareWorkspaces(self, left, right):
         for i in range(left.getNumberHistograms()):
             self.assertTrue(np.array_equal(left.dataY(i), right.dataY(i)))

Code/Mantid/instrument/nexusdictionaries/poldi.dic

@@ -1,12 +1,9 @@
-data=/entry1/detector/counts
-x-axis=/entry1/detector/time_binning
+data=/entry1/detector_1/counts
+x-axis=/entry1/detector_1/time_binning
 x-axis-name=time-of-flight
 title=/entry1/title
-det2theta=/entry1/detector/2theta
 sample=/entry1/sample/name
-sample_distance=/entry1/sample/distance1
 chopperspeed=/entry1/POLDI/chopper/rotation_speed
-monitor=/entry1/POLDI/detector/monitor
 #
 #
 FacilityName=/entry1/POLDI/SINQ/name
@@ -16,38 +13,20 @@ InstrumentName=/entry1/POLDI/name
 #
 ExperimentName=/entry1/title
 StartTime=/entry1/start_time
-#
-#ProtonMonitor=/entry1/proton_monitor/data
-#
-DetdRes=/entry1/POLDI/detector/d_resolution
-DetRadius=/entry1/POLDI/detector/det_radius
-DetDistance=/entry1/POLDI/detector/distance
-DetMonitor=/entry1/POLDI/detector/monitor
-DetPreset=/entry1/POLDI/detector/preset
-DetScattAngle=/entry1/POLDI/detector/scatt_angle
-DetTime=/entry1/POLDI/detector/time
-DetTwoTheta=/entry1/POLDI/detector/twotheta
-Detx0=/entry1/POLDI/detector/x0
-Dety0=/entry1/POLDI/detector/y0
-#
-Diaphragm1Dist=/entry1/POLDI/diaphragm1/distance
-Diaphragm1x=/entry1/POLDI/diaphragm1/x
-Diaphragm1x0=/entry1/POLDI/diaphragm1/x_zero
-Diaphragm1y=/entry1/POLDI/diaphragm1/y
-Diaphragm1y0=/entry1/POLDI/diaphragm1/y_zero
-#
-Diaphragme2Dist=/entry1/POLDI/diaphragm2/distance
-Diaphragme2xminus=/entry1/POLDI/diaphragm2/x_minus
-Diaphragme2xplus=/entry1/POLDI/diaphragm2/x_plus
-Diaphragme2xminus0=/entry1/POLDI/diaphragm2/xminus_zero
-Diaphragme2xplus0=/entry1/POLDI/diaphragm2/xplus_zero
-Diaphragme2zminus=/entry1/POLDI/diaphragm2/z_minus
-Diaphragme2zminus0=/entry1/POLDI/diaphragm2/z_minus_zero
-Diaphragme2plus=/entry1/POLDI/diaphragm2/z_plus
-Diaphragme2plus0=/entry1/POLDI/diaphragm2/zplus_zero
+DetRadius=/entry1/POLDI/detector_1/det_radius
+DetDistance=/entry1/POLDI/detector_1/distance
+DetMonitor=/entry1/POLDI/detector_1/monitor
+DetPreset=/entry1/POLDI/detector_1/preset
+DetScattAngle=/entry1/POLDI/detector_1/twotheta_detector
+DetTime=/entry1/POLDI/detector_1/time
+DetTwoTheta=/entry1/POLDI/detector_1/polar_angle
+Detx0=/entry1/POLDI/detector_1/x_pixel_offset
+Dety0=/entry1/POLDI/detector_1/y_pixel_offset
+
 #
 ChopperName=/entry1/POLDI/chopper/name
-ChopperPhase=/entry1/POLDI/chopper/chopper_phase
+ChopperDistance=/entry1/POLDI/chopper/distance
+ChopperParkingPosition=/entry1/POLDI/chopper/parking_position
 ChopperSpeed=/entry1/POLDI/chopper/rotation_speed
 
 # There seems to be an issue with this field in some data files.
@@ -57,7 +36,3 @@ ChopperSpeed=/entry1/POLDI/chopper/rotation_speed
 SampleName=/entry1/POLDI/name
 #
 start_time=/entry1/start_time
-#
-TablePositionX=/entry1/sample/sample_x
-TablePositionY=/entry1/sample/sample_y
-TablePositionZ=/entry1/sample/sample_lift

Code/Mantid/instrument/nexusdictionaries/poldi_legacy.dic

@@ -0,0 +1,66 @@
+# This file defines the format of POLDI files before the 2015 cycle.
+# For the currently valid definition see poldi.dic in this directory.
+#
+data=/entry1/detector/counts
+x-axis=/entry1/detector/time_binning
+x-axis-name=time-of-flight
+title=/entry1/title
+det2theta=/entry1/detector/2theta
+sample=/entry1/sample/name
+sample_distance=/entry1/sample/distance1
+chopperspeed=/entry1/POLDI/chopper/rotation_speed
+monitor=/entry1/POLDI/detector/monitor
+#
+#
+FacilityName=/entry1/POLDI/SINQ/name
+FacilityType=/entry1/POLDI/SINQ/type
+#
+InstrumentName=/entry1/POLDI/name
+#
+ExperimentName=/entry1/title
+StartTime=/entry1/start_time
+#
+#ProtonMonitor=/entry1/proton_monitor/data
+#
+DetdRes=/entry1/POLDI/detector/d_resolution
+DetRadius=/entry1/POLDI/detector/det_radius
+DetDistance=/entry1/POLDI/detector/distance
+DetMonitor=/entry1/POLDI/detector/monitor
+DetPreset=/entry1/POLDI/detector/preset
+DetScattAngle=/entry1/POLDI/detector/scatt_angle
+DetTime=/entry1/POLDI/detector/time
+DetTwoTheta=/entry1/POLDI/detector/twotheta
+Detx0=/entry1/POLDI/detector/x0
+Dety0=/entry1/POLDI/detector/y0
+#
+Diaphragm1Dist=/entry1/POLDI/diaphragm1/distance
+Diaphragm1x=/entry1/POLDI/diaphragm1/x
+Diaphragm1x0=/entry1/POLDI/diaphragm1/x_zero
+Diaphragm1y=/entry1/POLDI/diaphragm1/y
+Diaphragm1y0=/entry1/POLDI/diaphragm1/y_zero
+#
+Diaphragme2Dist=/entry1/POLDI/diaphragm2/distance
+Diaphragme2xminus=/entry1/POLDI/diaphragm2/x_minus
+Diaphragme2xplus=/entry1/POLDI/diaphragm2/x_plus
+Diaphragme2xminus0=/entry1/POLDI/diaphragm2/xminus_zero
+Diaphragme2xplus0=/entry1/POLDI/diaphragm2/xplus_zero
+Diaphragme2zminus=/entry1/POLDI/diaphragm2/z_minus
+Diaphragme2zminus0=/entry1/POLDI/diaphragm2/z_minus_zero
+Diaphragme2plus=/entry1/POLDI/diaphragm2/z_plus
+Diaphragme2plus0=/entry1/POLDI/diaphragm2/zplus_zero
+#
+ChopperName=/entry1/POLDI/chopper/name
+ChopperPhase=/entry1/POLDI/chopper/chopper_phase
+ChopperSpeed=/entry1/POLDI/chopper/rotation_speed
+
+# There seems to be an issue with this field in some data files.
+# Once this is fixed, it can be reactivated.
+#ChopperSpeedTarget=/entry1/POLDI/chopper/rotation_speed_target
+#
+SampleName=/entry1/POLDI/name
+#
+start_time=/entry1/start_time
+#
+TablePositionX=/entry1/sample/sample_x
+TablePositionY=/entry1/sample/sample_y
+TablePositionZ=/entry1/sample/sample_lift