Skip to content

Commit

Permalink
Refs #11116 Moved more into MDReductionWrapper class.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ianbush
ianbush committed Feb 18, 2015
1 parent f91f732 commit d075c03
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 106 deletions.
133 changes: 108 additions & 25 deletions Code/Mantid/scripts/Inelastic/Direct/MDReductionWrapper.py
View file
@@ -1,3 +1,6 @@
import os
import time

from ReductionWrapper import *

class MDReductionWrapper(ReductionWrapper):
Expand All @@ -7,40 +10,45 @@ class MDReductionWrapper(ReductionWrapper):
def __init__(self, instrumentName, web_var = None):
super(MDReductionWrapper, self).__init__(instrumentName, web_var = None)

# Create a file with the filename and path, with a '.lock' extension, to indicate file in use,
# unless this already exists, in which case do nothing and return False
def lock_obtained(self, fname):
"""Create a file with the filename and path, with a '.lock' extension, to indicate
file in use, unless this already exists, in which case do nothing and return False
"""
if not os.path.isfile(fname + '.lock'):
lock_file = open(fname + '.lock', 'w')
lock_file.close()
return True
else:
return False

# Remove lock file, exception will be thrown if this does not exist
def release_lock(self, fname):
"""Remove lock file, exception will be thrown if this does not exist
"""
os.remove(fname + '.lock')

# The table below shows the valid combinations for controlling the run numbers.
# N = number of accumulated MD files to produce (e.g. Run Range Starts is a list of N integers).
# _______________
# ____________________________|_1_|_2_|_3_|_4_|
# | 'Run Range Starts' | N | N | N | 1 |
# | 'Run Range Ends' | N | - | - | - |
# | 'Number of Runs to Merge' | - | N | 1 | 1 |
# ---------------------------------------------
#
# If file_run_number is between range_starts and range_ends return the range and MD file number (0...N-1),
# as a tuple or return -1, -1, -1 if the run number is not in a range asked for
def get_file_number(self, file_run_number, range_starts, range_ends):
""" The table below shows the valid combinations for controlling the run numbers.
N = number of accumulated MD files to produce (e.g. Run Range Starts is a list of N integers).
_______________
____________________________|_1_|_2_|_3_|_4_|
| 'Run Range Starts' | N | N | N | 1 |
| 'Run Range Ends' | N | - | - | - |
| 'Number of Runs to Merge' | - | N | 1 | 1 |
---------------------------------------------
If file_run_number is between range_starts and range_ends return the range and MD file number (0...N-1),
as a tuple or return -1, -1, -1 if the run number is not in a range asked for
"""

for i in range(0, len(range_starts)):
if ((file_run_number >= range_starts[i]) and (file_run_number <= range_ends[i])):
return i, range_starts[i], range_ends[i]
return -1, -1, -1

# If number_of_runs_to_merge is given return the run range and MD file number (0...N-1) as a tuple,
# or return -1, -1, -1 if the range is not as asked for
def get_file_number_alternative(self, file_run_number, range_starts, number_of_runs_to_merge):
""" If number_of_runs_to_merge is given return the run range and MD file number (0...N-1) as a tuple,
or return -1, -1, -1 if the range is not as asked for
"""
if (len(range_starts) == 1 and len(number_of_runs_to_merge) == 1):
if (file_run_number < range_starts[0]):
return -1, -1, -1
Expand All @@ -60,23 +68,25 @@ def get_file_number_alternative(self, file_run_number, range_starts, number_of_r
raise Exception("Size of range_starts, ", len(range_starts), " and number_of_runs_to_merge, ",
len(number_of_runs_to_merge), " are incompatible")

# If MD:Filenames is set return the appropriate filename, else return a filename such as 123_456_SQW.nxs
def get_file_name(self, file_number, start, end):
""" If MD:Filenames is set return the appropriate filename, else return a filename such as 123_456_SQW.nxs
"""
file_names = self._wvs.advanced_vars.get('MD:Filenames')
if (file_names is None or len(file_names) == 0):
return str(start) + '_' + str(end) + '_SQW.nxs'
else:
return file_names[file_number]

# For psi allowed combinations are shown below
# N = number of accumulated MD files to produce (e.g. Psi Starts is a list of N integers).
# ___________
# ___________________|_1_|_2_|_3_|
# | 'Psi Starts' | N | N | 1 |
# | 'Psi Increments' | 1 | N | 1 |
# --------------------------------
#
def get_psi(self, file_number, file_run_number, start, end):
""" For psi allowed combinations are shown below
N = number of accumulated MD files to produce (e.g. Psi Starts is a list of N integers).
___________
___________________|_1_|_2_|_3_|
| 'Psi Starts' | N | N | 1 |
| 'Psi Increments' | 1 | N | 1 |
--------------------------------
"""

psi_starts = self._wvs.advanced_vars.get('MD:Psi Starts')
psi_increments = self._wvs.advanced_vars.get('MD:Psi Increments')

Expand All @@ -90,6 +100,79 @@ def get_psi(self, file_number, file_run_number, start, end):
raise Exception("Size of psi_starts, ", len(psi_starts), " and psi_increments, ",
len(psi_increments), " are incompatible")

def convert_and_merge(self, input_file, input_dir, file_run_number):
""" Does the conversion process to a filebacked MD file, first working out the file
names required
"""
# Process ranges to merge into something more useful
range_starts = self._wvs.advanced_vars.get('MD:Run Range Starts')
range_ends = self._wvs.advanced_vars.get('MD:Run Range Ends')
number_of_runs_to_merge = self._wvs.advanced_vars.get('MD:Number of Runs to Merge')

if (len(range_ends) == 0):
file_number, start, end = self.get_file_number_alternative(file_run_number, range_starts, number_of_runs_to_merge)
else:
file_number, start, end = self.get_file_number(file_run_number, range_starts, range_ends)

merged_filename = input_dir + '/../' + self.get_file_name(file_number, start, end)

psi = self.get_psi(file_number, file_run_number, start, end)

ub_matrix = self._wvs.advanced_vars.get('MD:UB Matrix')

# Convert to an MD workspace and merge into the big file
loaded_ws = Load(Filename = input_file)

# UB matrix contains lattice and the beam direction
SetUB(Workspace = loaded_ws, a = ub_matrix[0], b = ub_matrix[1], c = ub_matrix[2])

# Add crystal rotation logging
# str(float(psi)) psi and can be saved as a number
AddSampleLog(Workspace = loaded_ws, LogName = 'Psi', LogText = str(float(psi)), LogType = 'Number')

# Set crystal rotation
SetGoniometer(Workspace = loaded_ws, Axis0 = 'Psi, 0, 1, 0, 1')

# Convert to MD
# If these values need to be changed they can be set in def_advanced_properties
# on the reduction class
pars = dict();
pars['InputWorkspace']='loaded_ws'
pars['QDimensions']='Q3D'
pars['dEAnalysisMode']='Direct'
pars['Q3DFrames']='HKL'
pars['QConversionScales']='HKL'
pars['PreprocDetectorsWS']='preprDetMantid'
pars['MinValues']=self._wvs.advanced_vars.get('MD:Minimum Extents')
pars['MaxValues']=self._wvs.advanced_vars.get('MD:Maximum Extents')
pars['SplitInto']=50
pars['MaxRecursionDepth']=1
pars['MinRecursionDepth']=1
pars['OutputWorkspace'] = 'merged_ws'

# Get the lock on the output file. Can obtain this whether or not the file exists yet.
# If lock is not obtained sleep for 10s. TODO: should this timeout?
# In case of problems will need to delete the lock file manually.
while(not self.lock_obtained(merged_filename)):
print "Waiting for lock on ", merged_filename
time.sleep(10)

# Load the output file if it exists, create it otherwise
if (os.path.exists(merged_filename)):
merged_ws = LoadMD(Filename = merged_filename, FileBackEnd = True)
pars['OverwriteExisting'] = False
else:
pars['OverwriteExisting'] = True

merged_ws = ConvertToMD(**pars)

# Save the files
if pars.get('OverwriteExisting'):
SaveMD(merged_ws, Filename = merged_filename, MakeFileBacked = True)
else:
SaveMD(merged_ws, Filename = merged_filename, UpdateFileBackEnd = True)

# Release the file lock now.
self.release_lock(merged_filename)


87 changes: 6 additions & 81 deletions Code/Mantid/scripts/test/MerlinAccumulateMDReduction.py
View file
@@ -1,11 +1,8 @@
""" MERLIN reduction script which also creates Merged MD files """

import time
import os

### Only required for running locally ###
import sys
sys.path.insert(0,'/home/whb43145/Mantid/mantid-develop/Code/Mantid/scripts/Inelastic')
#import sys
#sys.path.insert(0,'/home/whb43145/Mantid/mantid-develop/Code/Mantid/scripts/Inelastic')
#sys.path.insert(0,'/home/whb43145/autoreduction_test_production')

from Direct.MDReductionWrapper import *
Expand Down Expand Up @@ -67,7 +64,6 @@ def def_advanced_properties(self):
def main(self, input_file = None, output_directory = None, output_file = None):
# run reduction, write auxiliary script to add something here.
outWS = ReductionWrapper.reduce(input_file, output_directory)
#SaveNexus(outWS, Filename = output_file)

# When run from web service, instead return additional path for web server to copy data to
return outWS
Expand All @@ -76,93 +72,22 @@ def __init__(self, web_var = None):
""" sets properties defaults for the instrument with Name"""
super(ReduceMERLIN, self).__init__('MER',web_var)

def convert_and_merge(rd, input_file, input_dir, file_run_number):

# Process ranges to merge into something more useful
range_starts = web_var.advanced_vars.get('MD:Run Range Starts')
range_ends = web_var.advanced_vars.get('MD:Run Range Ends')
number_of_runs_to_merge = web_var.advanced_vars.get('MD:Number of Runs to Merge')

if (len(range_ends) == 0):
file_number, start, end = rd.get_file_number_alternative(file_run_number, range_starts, number_of_runs_to_merge)
else:
file_number, start, end = rd.get_file_number(file_run_number, range_starts, range_ends)

merged_filename = input_dir + '/../' + rd.get_file_name(file_number, start, end)

psi = rd.get_psi(file_number, file_run_number, start, end)

ub_matrix = web_var.advanced_vars.get('MD:UB Matrix')

# Convert to an MD workspace and merge into the big file
loaded_ws = Load(Filename = input_file)

# UB matrix contains lattice and the beam direction
SetUB(Workspace = loaded_ws, a = ub_matrix[0], b = ub_matrix[1], c = ub_matrix[2])

# Add crystal rotation logging
# str(float(psi)) psi and can be saved as a number
AddSampleLog(Workspace = loaded_ws, LogName = 'Psi', LogText = str(float(psi)), LogType = 'Number')

# Set crystal rotation
SetGoniometer(Workspace = loaded_ws, Axis0 = 'Psi, 0, 1, 0, 1')

# Convert to MD
# If these values need to be changed they can be set in def_advanced_properties
# on the reduction class
pars = dict();
pars['InputWorkspace']='loaded_ws'
pars['QDimensions']='Q3D'
pars['dEAnalysisMode']='Direct'
pars['Q3DFrames']='HKL'
pars['QConversionScales']='HKL'
pars['PreprocDetectorsWS']='preprDetMantid'
pars['MinValues']=web_var.advanced_vars.get('MD:Minimum Extents')
pars['MaxValues']=web_var.advanced_vars.get('MD:Maximum Extents')
pars['SplitInto']=50
pars['MaxRecursionDepth']=1
pars['MinRecursionDepth']=1
pars['OutputWorkspace'] = 'merged_ws'

# Get the lock on the output file. Can obtain this whether or not the file exists yet.
# If lock is not obtained sleep for 10s. TODO: should this timeout?
# In case of problems will need to delete the lock file manually.
while(not rd.lock_obtained(merged_filename)):
print "Waiting for lock on ", merged_filename
time.sleep(10)

# Load the output file if it exists, create it otherwise
if (os.path.exists(merged_filename)):
merged_ws = LoadMD(Filename = merged_filename, FileBackEnd = True)
pars['OverwriteExisting'] = False
else:
pars['OverwriteExisting'] = True

merged_ws = ConvertToMD(**pars)

# Save the files
if pars.get('OverwriteExisting'):
SaveMD(merged_ws, Filename = merged_filename, MakeFileBacked = True)
else:
SaveMD(merged_ws, Filename = merged_filename, UpdateFileBackEnd = True)

# Release the file lock now.
rd.release_lock(merged_filename)

# This is called by the autoreduction script itself
def reduce(input_file, output_dir):
# Define any extra directories needed for maps files etc. (not needed for running through autoreduction server)

### Only required for running locally ###
maps_dir = '/home/whb43145/Mantid/RAW_to_SPE/MERLIN/masks'
#maps_dir = '/home/whb43145/Mantid/RAW_to_SPE/MERLIN/masks'

input_path, input_filename = os.path.split(input_file)

# Merlin data starts in the format: 'MERXXXXX.raw', where XXXXXX is the run number
file_run_number = int(input_filename.split('MER')[1].split('.')[0])

### Only required for running locally ###
config.setDataSearchDirs('{0};{1};{2}'.format(input_path,output_dir,maps_dir))
#config.setDataSearchDirs('{0};{1};{2}'.format(input_path,output_dir,maps_dir))

config['defaultsave.directory'] = output_dir.encode('ascii','replace') # folder to save resulting spe/nxspe files

rd = ReduceMERLIN(web_var)
Expand All @@ -180,7 +105,7 @@ def reduce(input_file, output_dir):
outWS = rd.reduce(input_file, output_dir)

if (web_var.advanced_vars.get('MD:Accumulate to MD file')):
convert_and_merge(rd, output_filename, output_dir, file_run_number)
rd.convert_and_merge(output_filename, output_dir, file_run_number)

return None # Can make this an additional save directory

Expand Down

0 comments on commit d075c03

Please sign in to comment.