diff --git a/Code/Mantid/scripts/Inelastic/Direct/PropertyManager.py b/Code/Mantid/scripts/Inelastic/Direct/PropertyManager.py index 987c33499bb5..57ac4ee683b9 100644 --- a/Code/Mantid/scripts/Inelastic/Direct/PropertyManager.py +++ b/Code/Mantid/scripts/Inelastic/Direct/PropertyManager.py @@ -270,7 +270,7 @@ def set_input_parameters_ignore_nan(self,**kwargs): for par_name,value in kwargs.items() : - if not(value is None): + if not(value is None) and not par_name.startswith("MD:"): setattr(self,par_name,value) @@ -290,7 +290,8 @@ def set_input_parameters(self,**kwargs): for par_name,value in kwargs.items() : - setattr(self,par_name,value) + if not par_name.startswith("MD:"): + setattr(self,par_name,value) return self.getChangedProperties() diff --git a/Code/Mantid/scripts/test/MerlinAccumulateMDReduction.py b/Code/Mantid/scripts/test/MerlinAccumulateMDReduction.py new file mode 100644 index 000000000000..ab81e7d1a167 --- /dev/null +++ b/Code/Mantid/scripts/test/MerlinAccumulateMDReduction.py @@ -0,0 +1,291 @@ +""" MERLIN reduction script which also creates Merged MD files """ + +import time +import os + +### Only required for running locally ### +#import sys +#sys.path.insert(0,'/opt/Mantid/scripts/Inelastic') +#sys.path.insert(0,'/home/whb43145/autoreduction_test_production') + +from ReductionWrapper import * + +try: + import reduce_vars as web_var +except: + web_var = None + + +class ReduceMERLIN(ReductionWrapper): + @MainProperties + def def_main_properties(self, file_run_number=None): + """ Define main properties used in reduction """ + prop = {} + if file_run_number is not None: + prop['sample_run'] = file_run_number + else: + prop['sample_run'] = 22413 + + prop['wb_run'] = 23012 + prop['incident_energy'] = 15 + prop['energy_bins'] = [-0.1,0.00175,0.95] + + # Absolute units reduction properties. + prop['monovan_run'] = 22932 + prop['sample_mass'] = 10 + prop['sample_rmm'] = 435.96 + prop['bleed'] = False + return prop + + @AdvancedProperties + def def_advanced_properties(self): + """ Define advancded (optional) properties """ + prop = {}; + prop['map_file'] = 'one2one_125.map' + prop['monovan_mapfile'] = 'rings_125.map' + prop['hard_mask_file'] = 'Bjorn_mask.msk' + prop['det_cal_file'] = 'det_corr_125.dat' + prop['data_file_ext'] = '.nxs' + prop['save_format'] = 'nxspe' + + # MD accumulation properties - prefix 'MD:' stops these parameters being + # passed through to the initial reduction step. + prop['MD:Accumulate to MD file'] = True + prop['MD:Run Range Starts'] = [22413] + prop['MD:Run Range Ends'] = [22932] + prop['MD:Number of Runs to Merge'] = [] + prop['MD:Psi Starts'] = [0] + prop['MD:Psi Increments'] = [2] + prop['MD:UB Matrix'] = [1.4165, 1.4165, 1.4165] # a, b, c + prop['MD:Filenames'] = None + prop['MD:Minimum Extents'] = '-3, -5, -4, -5.0' + prop['MD:Maximum Extents'] = '5, 2, 4, 30.0' + + return prop + + @iliad + def main(self, input_file = None, output_directory = None, output_file = None): + # run reduction, write auxiliary script to add something here. + outWS = ReductionWrapper.reduce(input_file, output_directory) + #SaveNexus(outWS, Filename = output_file) + + # When run from web service, instead return additional path for web server to copy data to + return outWS + + def __init__(self, web_var = None): + """ sets properties defaults for the instrument with Name""" + ReductionWrapper.__init__(self,'MER',web_var) + +# Create a file with the filename and path, with a '.lock' extension, to indicate file in use, +# unless this already exists, in which case do nothing and return False +def lock_obtained(fname): + if not os.path.isfile(fname + '.lock'): + lock_file = open(fname + '.lock', 'w') + lock_file.close() + return True + else: + return False + +# Remove lock file, exception will be thrown if this does not exist +def release_lock(fname): + os.remove(fname + '.lock') + + +# The table below shows the valid combinations for controlling the run numbers. +# N = number of accumulated MD files to produce (e.g. Run Range Starts is a list of N integers). +# _______________ +# ____________________________|_1_|_2_|_3_|_4_| +# | 'Run Range Starts' | N | N | N | 1 | +# | 'Run Range Ends' | N | - | - | - | +# | 'Number of Runs to Merge' | - | N | 1 | 1 | +# --------------------------------------------- +# +# If file_run_number is between range_starts and range_ends return the range and MD file number (0...N-1), +# as a tuple or return -1, -1, -1 if the run number is not in a range asked for +def get_file_number(file_run_number, range_starts, range_ends): + for i in range(0, len(range_starts)): + if ((file_run_number >= range_starts[i]) and (file_run_number <= range_ends[i])): + return i, range_starts[i], range_ends[i] + return -1, -1, -1 + +# If number_of_runs_to_merge is given return the run range and MD file number (0...N-1) as a tuple, +# or return -1, -1, -1 if the range is not as asked for +def get_file_number_alternative(file_run_number, range_starts, number_of_runs_to_merge): + if (len(range_starts) == 1 and len(number_of_runs_to_merge) == 1): + if (file_run_number < range_starts[0]): + return -1, -1, -1 + file_number = file_run_number//number_of_runs_to_merge[0] - range_starts[0]//number_of_runs_to_merge[0] + return file_number + 1, file_number * number_of_runs_to_merge[0], (file_number + 1) * number_of_runs_to_merge[0] + elif (len(range_starts) > 1 and len(number_of_runs_to_merge) == 1): + for i in range(0, len(range_starts)): + if (file_run_number >= range_starts[i] and file_run_number < range_starts[i] + number_of_runs_to_merge[0]): + return i, range_starts[i], range_starts[i] + number_of_runs_to_merge[0] - 1 + return -1, -1, -1 + elif (len(range_starts) == len(number_of_runs_to_merge)): + for i in range(0, len(range_starts)): + if (file_run_number >= range_starts[i] and file_run_number < range_starts[i] + number_of_runs_to_merge[i]): + return i, range_starts[i], range_starts[i] + number_of_runs_to_merge[i] - 1 + return -1, -1, -1 + + raise Exception("Size of range_starts, ", len(range_starts), " and number_of_runs_to_merge, ", + len(number_of_runs_to_merge), " are incompatible") + +# If MD:Filenames is set return the appropriate filename, else return a filename such as 123_456_SQW.nxs +def get_file_name(file_number, start, end): + file_names = web_var.advanced_vars.get('MD:Filenames') + if (file_names is None or len(file_names) == 0): + return str(start) + '_' + str(end) + '_SQW.nxs' + else: + return file_names[file_number] + +# For psi allowed combinations are shown below +# N = number of accumulated MD files to produce (e.g. Psi Starts is a list of N integers). +# ___________ +# ___________________|_1_|_2_|_3_| +# | 'Psi Starts' | N | N | 1 | +# | 'Psi Increments' | 1 | N | 1 | +# -------------------------------- +# +def get_psi(file_number, file_run_number, start, end): + psi_starts = web_var.advanced_vars.get('MD:Psi Starts') + psi_increments = web_var.advanced_vars.get('MD:Psi Increments') + + if (len(psi_starts) == 1 and len(psi_increments) == 1): + return psi_starts[0] + (file_run_number - start) * psi_increments[0] + elif (len(psi_starts) == len(psi_increments)): + return psi_starts[file_number] + (file_run_number - start) * psi_increments[file_number] + elif (len(psi_starts) > 1 and len(psi_increments) == 1): + return psi_starts[file_number] + (file_run_number - start) * psi_increments[0] + else: + raise Exception("Size of psi_starts, ", len(psi_starts), " and psi_increments, ", + len(psi_increments), " are incompatible") + + +def convert_and_merge(input_file, output_file, psi, ub_matrix, file_run_number): + # Convert to an MD workspace and merge into the big file + loaded_ws = Load(Filename = input_file) + + # UB matrix contains lattice and the beam direction + SetUB(Workspace = loaded_ws, a = ub_matrix[0], b = ub_matrix[1], c = ub_matrix[2]) + + # Add crystal rotation logging + # str(float(psi)) psi and can be saved as a number + AddSampleLog(Workspace = loaded_ws, LogName = 'Psi', LogText = str(float(psi)), LogType = 'Number') + + # Set crystal rotation + SetGoniometer(Workspace = loaded_ws, Axis0 = 'Psi, 0, 1, 0, 1') + + # Convert to MD + # If these values need to be changed they can be set in def_advanced_properties + # on the reduction class + pars = dict(); + pars['InputWorkspace']='loaded_ws' + pars['QDimensions']='Q3D' + pars['dEAnalysisMode']='Direct' + pars['Q3DFrames']='HKL' + pars['QConversionScales']='HKL' + pars['PreprocDetectorsWS']='preprDetMantid' + pars['MinValues']=web_var.advanced_vars.get('MD:Minimum Extents') + pars['MaxValues']=web_var.advanced_vars.get('MD:Maximum Extents') + pars['SplitInto']=50 + pars['MaxRecursionDepth']=1 + pars['MinRecursionDepth']=1 + pars['OutputWorkspace'] = 'merged_ws' + + # Get the lock on the output file. Can obtain this whether or not the file exists yet. + # If lock is not obtained sleep for 10s. TODO: should this timeout? + # In case of problems will need to delete the lock file manually. + while(not lock_obtained(output_file)): + print "Waiting for lock on ", output_file + time.sleep(10) + + # Load the output file if it exists, create it otherwise + if (os.path.exists(output_file)): + merged_ws = LoadMD(Filename = output_file, FileBackEnd = True) + pars['OverwriteExisting'] = False + else: + pars['OverwriteExisting'] = True + + merged_ws = ConvertToMD(**pars) + + # Save the file + print output_file + if pars.get('OverwriteExisting'): + SaveMD(merged_ws, Filename = output_file, MakeFileBacked = True) + else: + SaveMD(merged_ws, Filename = output_file, UpdateFileBackEnd = True) + + # Release the file lock now. + release_lock(output_file) + +# This is called by the autoreduction script itself +def reduce(input_file, output_dir): + # Define any extra directories needed for maps files etc. (not needed for running through autoreduction server) + + ### Only required for running locally ### + #maps_dir = '/home/whb43145/autoreduction_test_russell/maps' + + input_path, input_filename = os.path.split(input_file) + + # Merlin data starts in the format: 'MERXXXXX.raw', where XXXXXX is the run number + file_run_number = int(input_filename.split('MER')[1].split('.')[0]) + + config.setDataSearchDirs('{0};{1};{2}'.format(input_path,output_dir,maps_dir)) + config['defaultsave.directory'] = output_dir.encode('ascii','replace') # folder to save resulting spe/nxspe files + + rd = ReduceMERLIN(web_var) + rd._run_from_web = False + rd.def_advanced_properties() + rd.def_main_properties(file_run_number) + + if not rd._run_from_web: + run_dir=os.path.dirname(os.path.realpath(__file__)) + file = os.path.join(run_dir,'reduce_vars.py') + rd.save_web_variables(file); + + output_filename = output_dir + '/' + rd.reducer.prop_man.save_file_name + '.nxspe' + + outWS = rd.reduce(input_file, output_dir) + + if (web_var.advanced_vars.get('MD:Accumulate to MD file')): + # Process ranges to merge into something more useful + range_starts = web_var.advanced_vars.get('MD:Run Range Starts') + range_ends = web_var.advanced_vars.get('MD:Run Range Ends') + number_of_runs_to_merge = web_var.advanced_vars.get('MD:Number of Runs to Merge') + + if (len(range_ends) == 0): + file_number, start, end = get_file_number_alternative(file_run_number, range_starts, number_of_runs_to_merge) + else: + file_number, start, end = get_file_number(file_run_number, range_starts, range_ends) + + merged_filename = output_dir + '/../' + get_file_name(file_number, start, end) + + psi = get_psi(file_number, file_run_number, start, end) + + ub_matrix = web_var.advanced_vars.get('MD:UB Matrix') + + convert_and_merge(output_filename, merged_filename, psi, ub_matrix, file_run_number) + + return None # Can make this an additional save directory + + +def main(*args, **kwargs): + if not kwargs and sys.argv: #If called from command line + if len(sys.argv) == 3 and '=' not in sys.argv: + # with two simple inputs + kwargs = { 'data' : sys.argv[1], 'output':sys.argv[2]} + else: + # With key value inputs + kwargs = dict(x.split('=', 1) for x in sys.argv[1:]) + if not kwargs and 'data' not in kwargs and 'output' not in kwargs: + raise ValueError("Data and Output paths must be supplied") + + # If additional storage location are required, return them as a list of accessible directory paths + additional_save_location = reduce(kwargs['data'], kwargs['output']) + return additional_save_location + + +if __name__=="__main__": + main() + + diff --git a/Code/Mantid/scripts/test/fake_runner.py b/Code/Mantid/scripts/test/fake_runner.py new file mode 100644 index 000000000000..8ffefca74fac --- /dev/null +++ b/Code/Mantid/scripts/test/fake_runner.py @@ -0,0 +1,33 @@ +""" Allows autoreduction runs to be kicked off one after another, for testing MD file accumulation """ + +import sched, time +import subprocess + +s = sched.scheduler(time.time, time.sleep) + +def print_time(i): + print "From print_time", time.ctime(), " ", i + +def call_reduce(start_time, i, interval_time, kwargs): + start = time.time() + subprocess.call(['python', 'reduce.py', kwargs['data'], kwargs['output']]) + end = time.time() + print 'Reduce time = ', end - start, ', leaving ', start_time + (i+1)*interval_time - end, ' second interval.' + print 'Next run will start at:', time.ctime(start_time + (i+1)*interval_time), '. Last file run:', kwargs['data'] + +def print_some_times(): + time_to_sleep = 0 + interval_time = 60*8 # in seconds + run_start = 22413 + + start_time = time.time() + + for i in range(0, 518): # ending on 22931 + kwargs = {'data': '', 'output': '/home/whb43145/autoreduce_out'} + kwargs['data'] = '/isis/Instruments$/NDXMERLIN/Instrument/data/cycle_14_1/MER' + str(run_start+i).zfill(5) + '.nxs' + s.enter(i*interval_time, 1, call_reduce, (start_time, i, interval_time, kwargs)) + + s.run() + +print_some_times() +