From 3115009b7b02e1372b7d8296f50a48d7ebe9b36f Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 19:50:35 -0700 Subject: [PATCH 01/14] tweaks for Rayonix --- btx/processing/peak_finder.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/btx/processing/peak_finder.py b/btx/processing/peak_finder.py index 8485de4e1..a2929c0bd 100644 --- a/btx/processing/peak_finder.py +++ b/btx/processing/peak_finder.py @@ -65,6 +65,11 @@ def set_up_psana_interface(self, exp, run, det_type): # additional self variables for tracking peak stats self.iX = self.psi.det.indexes_x(self.psi.run).astype(np.int64) self.iY = self.psi.det.indexes_y(self.psi.run).astype(np.int64) + if det_type == 'Rayonix': + self.iX = np.expand_dims(self.iX, axis=0) + self.iY = np.expand_dims(self.iY, axis=0) + print(f"self.iX.shape = {self.iX.shape}") + self.ipx, self.ipy = self.psi.det.point_indexes(self.psi.run, pxy_um=(0, 0)) # retrieve clen from psana if None or a PV code is supplied @@ -136,7 +141,10 @@ def set_up_cxi(self, tag=''): # for storing images in crystFEL format det_shape = self.psi.det.shape() - dim0, dim1 = det_shape[0] * det_shape[1], det_shape[2] + if self.psi.det_type == 'Rayonix': + dim0, dim1 = det_shape[0], det_shape[1] + else: + dim0, dim1 = det_shape[0] * det_shape[1], det_shape[2] data_1 = entry_1.create_dataset('/entry_1/data_1/data', (self.n_events, dim0, dim1), chunks=(1, dim0, dim1), maxshape=(None, dim0, dim1),dtype=np.float32) data_1.attrs["axes"] = "experiment_identifier" From b77277dd944e23c3046deaeb8d0e1371f76b9c63 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 19:58:58 -0700 Subject: [PATCH 02/14] fixed docstring --- btx/interfaces/mask_interface.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/btx/interfaces/mask_interface.py b/btx/interfaces/mask_interface.py index bfd08d8c1..ec0a9f371 100644 --- a/btx/interfaces/mask_interface.py +++ b/btx/interfaces/mask_interface.py @@ -16,7 +16,7 @@ def __init__(self, exp, run, det_type): def generate_from_psana_run(self, thresholds, n_images=1): """ Generate a mask by extracting the first num_images from the indicated run, - thresholding each, and then merging them. + thresholding each, and then merging them. A value of 0 indicates a bad pixel. Parameters ---------- @@ -24,11 +24,6 @@ def generate_from_psana_run(self, thresholds, n_images=1): (lower, upper) thresholds for pixel value n_images : int number of images to threshold - - Returns - ------- - mask : numpy.ndarray, shape (n_panels, n_pixels_fs, n_pixels_ss) - binary mask, where 0 indicates a bad pixel """ # retrieve images from run images = self.psi.get_images(n_images, assemble=False) From 90dd0c620eed8d3f596cc1027aa894e98baaabc6 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 20:00:36 -0700 Subject: [PATCH 03/14] tweak for rayonix, which is 2d unassembled --- btx/diagnostics/run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/btx/diagnostics/run.py b/btx/diagnostics/run.py index 1ebcb2541..f8113e104 100644 --- a/btx/diagnostics/run.py +++ b/btx/diagnostics/run.py @@ -58,8 +58,9 @@ def finalize_powders(self): self.powders_final['max'] = np.max(powder_max, axis=0) self.powders_final['avg'] = powder_sum / float(total_n_proc) self.powders_final['std'] = np.sqrt(powder_sqr / float(total_n_proc) - np.square(self.powders_final['avg'])) - for key in self.powders_final.keys(): - self.powders_final[key] = assemble_image_stack_batch(self.powders_final[key], self.pixel_index_map) + if self.psi.det_type != 'Rayonix': + for key in self.powders_final.keys(): + self.powders_final[key] = assemble_image_stack_batch(self.powders_final[key], self.pixel_index_map) def save_powders(self, outdir): """ From ebe5a40a03983d74d1225030803d442615e9cf5b Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 21:38:35 -0700 Subject: [PATCH 04/14] adding a fetch_from_mrxv function --- btx/interfaces/mask_interface.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/btx/interfaces/mask_interface.py b/btx/interfaces/mask_interface.py index ec0a9f371..da37d985a 100644 --- a/btx/interfaces/mask_interface.py +++ b/btx/interfaces/mask_interface.py @@ -1,5 +1,8 @@ import numpy as np -import h5py, sys +import h5py +import os +import sys +import glob from btx.interfaces.psana_interface import * class MaskInterface: @@ -38,6 +41,33 @@ def generate_from_psana_run(self, thresholds, n_images=1): self.mask = mask else: self.mask *= mask + + def retrieve_from_mrxv(self, mrxv_path='/cds/sw/package/autosfx/mrxv/masks', dataset='/entry_1/data_1/mask'): + """ + Retrieve the latest mask from mrxv. + + Parameters + ---------- + dataset : str + internal path to dataset, only relevant for mask_format='crystfel' + """ + try: + mask_file = glob.glob(os.path.join(mrxv_path, f"{self.det_type}_latest.*"))[0] + assert os.path.isfile(mask_file) + print(f"Retrieving mask file {mask_file}") + except: + sys.exit("Detector type not yet available in mrxv") + + if h5py.is_hdf5(mask_file): + print("Crystfel mask detected") + mask_format = 'crystfel' + elif mask_file.split('.')[-1] == 'npy': + mask_format = 'psana' + else: + mask_format = 'cctbx' + + self.load_mask(mask_file, mask_format=mask_format, dataset=dataset) + assert self.psi.det.shape() == self.mask.shape def load_mask(self, input_file, mask_format='crystfel', dataset='/entry_1/data_1/mask'): """ From 59859356d4242934d197d2221cbb8f9c76c7e211 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 21:50:02 -0700 Subject: [PATCH 05/14] adding fetch_mask task --- scripts/tasks.py | 15 +++++++++++++++ tutorial/mfxlv4920.yaml | 28 ++++++++++++++++------------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/scripts/tasks.py b/scripts/tasks.py index 1c28f32bc..8ac6c5ff8 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -14,6 +14,21 @@ def test(config): requests.post(update_url, json=[ { "key": "root_dir", "value": f"{config.setup.root_dir}" }, { "key": "experiment_name", "value": "mfxp19619"} ]) +def fetch_mask(config): + from btx.interfaces.mask_interface import MaskInterface + setup = config.setup + task = config.fetch_mask + """ Fetch most recent mask for this detector from mrxv. """ + taskdir = os.path.join(setup.root_dir, 'mask') + os.makedirs(taskdir, exist_ok=True) + mi = MaskInterface(exp=setup.exp, + run=setup.run, + det_type=setup.det_type) + mi.retrieve_from_mrxv(mrxv_path=task.mrxv_path, dataset=task.dataset) + logger.info(f'Saving Mask to {taskdir}') + mi.save_mask(os.path.join(taskdir, f'r{setup.run:04}.npy')) + logger.debug('Done!') + def run_analysis(config): from btx.diagnostics.run import RunDiagnostics setup = config.setup diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index 7ccdf6a48..852575570 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -1,27 +1,31 @@ setup: root_dir: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/' exp: 'mfxlv4920' - run: 9 - det_type: 'epix10k2M' + run: 234 + det_type: 'Rayonix' + +fetch_mask: + mrxv_path: '/cds/home/a/apeck/mrxv/masks' + dataset: '/entry_1/data_1/mask' run_analysis: max_events: -1 - mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/mrxv_mask.npy' + mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/rayonix_mask.npy' opt_distance: - center: 834 832.5 - input_geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/4-end.data' + center: 960 960 + input_geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/176-end.data' find_peaks: tag: '' - mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/mrxv_mask.npy' - psana_mask: True - min_peaks: 2 + mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/rayonix_mask.npy' + psana_mask: False + min_peaks: 10 max_peaks: 2048 - npix_min: 1 + npix_min: 2 npix_max: 30 - amax_thr: 800. - atot_thr: 1000. + amax_thr: 300. + atot_thr: 600. son_min: 10.0 peak_rank: 3 r0: 3.0 @@ -30,7 +34,7 @@ find_peaks: index: tag: '' - geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/r0004-end.geom' + geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/geom/r0175.geom' int_radius: '4,5,6' methods: 'mosflm' cell: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/cco.cell' From 66a223b61a2572809d5573719b3d39e4df6d98e7 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 21:55:28 -0700 Subject: [PATCH 06/14] fixed nomenclature for fetched mask --- scripts/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tasks.py b/scripts/tasks.py index 8ac6c5ff8..58e5e0a77 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -26,7 +26,7 @@ def fetch_mask(config): det_type=setup.det_type) mi.retrieve_from_mrxv(mrxv_path=task.mrxv_path, dataset=task.dataset) logger.info(f'Saving Mask to {taskdir}') - mi.save_mask(os.path.join(taskdir, f'r{setup.run:04}.npy')) + mi.save_mask(os.path.join(taskdir, f'r0000.npy')) logger.debug('Done!') def run_analysis(config): From 59e2e62d14a96e5c6893e2cc041da4d1e413b619 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 22:08:40 -0700 Subject: [PATCH 07/14] updated yaml --- tutorial/mfxlv4920.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index 852575570..ee4c96676 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -1,7 +1,7 @@ setup: root_dir: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/' exp: 'mfxlv4920' - run: 234 + run: 175 det_type: 'Rayonix' fetch_mask: From b298dcdcca1162450ba3d2f3a4d100d0054f1cc0 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 22:21:53 -0700 Subject: [PATCH 08/14] adding a function to fetch latest mask or geom --- btx/misc/shortcuts.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/btx/misc/shortcuts.py b/btx/misc/shortcuts.py index 5c108d47d..f75b3aeb2 100644 --- a/btx/misc/shortcuts.py +++ b/btx/misc/shortcuts.py @@ -1,4 +1,6 @@ +import numpy as np import os +import glob class AttrDict(dict): """ Nested Attribute Dictionary @@ -29,3 +31,27 @@ def __getattr__(self, item): return self.__getitem__(item) except KeyError: raise AttributeError(item) + +def fetch_latest(fnames, run): + """ + Fetch the most recently created (in terms of run numbers) file. + Here we assume that files are named /{base_path}/r{run:04}.* . + + Parameters + ---------- + fnames : str + glob-expandable string pointing to geom or mask files + run : int + run number + + Returns + ------- + fname : str + filename of relevant geom or mask file + """ + fnames = glob.glob(fnames) + avail = [os.path.basename(f)[1:].split('.')[0] for f in fnames] + avail = np.array([int(a) for a in avail]) + sort_idx = np.argsort(avail) + idx = np.searchsorted(avail[sort_idx], run, side='right') - 1 + return fnames[sort_idx[idx]] From 0aa122fac385c3505d88c73a8375d90de446b5db Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sat, 30 Apr 2022 22:34:07 -0700 Subject: [PATCH 09/14] run_analysis now using fetch_latest for mask --- scripts/tasks.py | 5 ++++- tutorial/mfxlv4920.yaml | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/tasks.py b/scripts/tasks.py index 58e5e0a77..fb6a311b9 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -31,16 +31,19 @@ def fetch_mask(config): def run_analysis(config): from btx.diagnostics.run import RunDiagnostics + from btx.misc.shortcuts import fetch_latest setup = config.setup task = config.run_analysis """ Generate the max, avg, and std powders for a given run. """ taskdir = os.path.join(setup.root_dir, 'powder') os.makedirs(taskdir, exist_ok=True) + mask_file = fetch_latest(fnames=os.path.join(setup.root_dir, 'mask', 'r*.npy'),run=setup.run) + logger.debug(f'Applying mask: {mask_file}...') rd = RunDiagnostics(exp=setup.exp, run=setup.run, det_type=setup.det_type) logger.debug(f'Computing Powder for run {setup.run} of {setup.exp}...') - rd.compute_run_stats(max_events=task.max_events, mask=task.mask) + rd.compute_run_stats(max_events=task.max_events, mask=mask_file) logger.info(f'Saving Powders and plots to {taskdir}') rd.save_powders(taskdir) rd.visualize_powder(output=os.path.join(taskdir, f"powder_r{rd.psi.run:04}.png")) diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index ee4c96676..f4c960184 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -10,7 +10,6 @@ fetch_mask: run_analysis: max_events: -1 - mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/rayonix_mask.npy' opt_distance: center: 960 960 From 7911a2d0d9606cac21d9251437ca2e756765c69c Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sun, 1 May 2022 07:10:31 -0700 Subject: [PATCH 10/14] added a fetch_geom task --- btx/misc/metrology.py | 27 +++++++++++++++++++++++++++ scripts/tasks.py | 13 ++++++++++++- tutorial/mfxlv4920.yaml | 3 ++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/btx/misc/metrology.py b/btx/misc/metrology.py index c11a9bcdc..aa2bf87af 100644 --- a/btx/misc/metrology.py +++ b/btx/misc/metrology.py @@ -4,6 +4,33 @@ from psgeom import camera, sensors from btx.interfaces.psana_interface import PsanaInterface +def retrieve_from_mrxv(det_type, out_geom, mrxv_path='/cds/sw/package/autosfx/mrxv/geometries'): + """ + Fetch latest geom file for this detector from mrxv. Currently + assume that the file is in CrystFEL geom format. C + + Parameters + ---------- + det_type : str + detector name + mrxv_path : str + path to mrxv geom folder + """ + try: + in_geom = os.path.join(mrxv_path, f"{det_type}_latest.geom") + print(in_geom) + assert os.path.isfile(in_geom) + print(f"Retrieving geom file {in_geom}") + except: + sys.exit("Detector type not yet available in mrxv") + + if det_type == 'epix10k2M': + geom = camera.CompoundAreaCamera.from_crystfel_file(in_geom, element_type=sensors.Epix10kaSegment) + else: + geom = camera.CompoundAreaCamera.from_crystfel_file(in_geom) + + geom.to_crystfel_file(out_geom) + def modify_crystfel_header(input_file, output_file): """ Modify the header of a psgeom-generated Crystfel (.geom) file, diff --git a/scripts/tasks.py b/scripts/tasks.py index fb6a311b9..4117880a7 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -25,10 +25,21 @@ def fetch_mask(config): run=setup.run, det_type=setup.det_type) mi.retrieve_from_mrxv(mrxv_path=task.mrxv_path, dataset=task.dataset) - logger.info(f'Saving Mask to {taskdir}') + logger.info(f'Saving mrxv mask to {taskdir}') mi.save_mask(os.path.join(taskdir, f'r0000.npy')) logger.debug('Done!') +def fetch_geom(config): + from btx.misc.metrology import retrieve_from_mrxv + setup = config.setup + task = config.fetch_geom + """ Fetch latest geometry for this detector from mrxv. """ + taskdir = os.path.join(setup.root_dir, 'geom') + os.makedirs(taskdir, exist_ok=True) + logger.info(f'Saving mrxv geom to {taskdir}') + retrieve_from_mrxv(det_type=setup.det_type, out_geom=os.path.join(taskdir, f'r0000.geom')) + logger.debug('Done!') + def run_analysis(config): from btx.diagnostics.run import RunDiagnostics from btx.misc.shortcuts import fetch_latest diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index f4c960184..59cdac32d 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -5,9 +5,10 @@ setup: det_type: 'Rayonix' fetch_mask: - mrxv_path: '/cds/home/a/apeck/mrxv/masks' dataset: '/entry_1/data_1/mask' +fetch_geom: + run_analysis: max_events: -1 From 7427fb9a62d772a2703fa353b3f75096bcb62680 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sun, 1 May 2022 07:32:05 -0700 Subject: [PATCH 11/14] working for fetch_mask, fetch_geom, run_analysis, opt_distance --- btx/diagnostics/run.py | 2 +- scripts/tasks.py | 28 ++++++++++++++++------------ tutorial/mfxlv4920.yaml | 1 - 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/btx/diagnostics/run.py b/btx/diagnostics/run.py index f8113e104..02ceaab13 100644 --- a/btx/diagnostics/run.py +++ b/btx/diagnostics/run.py @@ -72,7 +72,7 @@ def save_powders(self, outdir): path to directory in which to save powders, optional """ for key in self.powders_final.keys(): - np.save(os.path.join(outdir, f"powder_{key}_r{self.psi.run:04}.npy"), self.powders_final[key]) + np.save(os.path.join(outdir, f"r{self.psi.run:04}_{key}.npy"), self.powders_final[key]) def compute_stats(self, img): """ diff --git a/scripts/tasks.py b/scripts/tasks.py index 4117880a7..8d57de34a 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -24,7 +24,7 @@ def fetch_mask(config): mi = MaskInterface(exp=setup.exp, run=setup.run, det_type=setup.det_type) - mi.retrieve_from_mrxv(mrxv_path=task.mrxv_path, dataset=task.dataset) + mi.retrieve_from_mrxv(dataset=task.dataset) logger.info(f'Saving mrxv mask to {taskdir}') mi.save_mask(os.path.join(taskdir, f'r0000.npy')) logger.debug('Done!') @@ -48,7 +48,8 @@ def run_analysis(config): """ Generate the max, avg, and std powders for a given run. """ taskdir = os.path.join(setup.root_dir, 'powder') os.makedirs(taskdir, exist_ok=True) - mask_file = fetch_latest(fnames=os.path.join(setup.root_dir, 'mask', 'r*.npy'),run=setup.run) + os.makedirs(os.path.join(taskdir, 'figs'), exist_ok=True) + mask_file = fetch_latest(fnames=os.path.join(setup.root_dir, 'mask', 'r*.npy'), run=setup.run) logger.debug(f'Applying mask: {mask_file}...') rd = RunDiagnostics(exp=setup.exp, run=setup.run, @@ -57,33 +58,36 @@ def run_analysis(config): rd.compute_run_stats(max_events=task.max_events, mask=mask_file) logger.info(f'Saving Powders and plots to {taskdir}') rd.save_powders(taskdir) - rd.visualize_powder(output=os.path.join(taskdir, f"powder_r{rd.psi.run:04}.png")) - rd.visualize_stats(output=os.path.join(taskdir, f"stats_r{rd.psi.run:04}.png")) + rd.visualize_powder(output=os.path.join(taskdir, f"figs/powder_r{rd.psi.run:04}.png")) + rd.visualize_stats(output=os.path.join(taskdir, f"figs/stats_r{rd.psi.run:04}.png")) logger.debug('Done!') def opt_distance(config): from btx.diagnostics.geom_opt import GeomOpt from btx.misc.metrology import modify_crystfel_header, generate_geom_file + from btx.misc.shortcuts import fetch_latest setup = config.setup task = config.opt_distance """ Optimize the detector distance from an AgBehenate run. """ taskdir = os.path.join(setup.root_dir, 'geom') os.makedirs(taskdir, exist_ok=True) + os.makedirs(os.path.join(taskdir, 'figs'), exist_ok=True) geom_opt = GeomOpt(exp=setup.exp, run=setup.run, det_type=setup.det_type) task.center = tuple([float(elem) for elem in task.center.split()]) logger.debug(f'Optimizing detector distance for run {setup.run} of {setup.exp}...') - dist = geom_opt.opt_distance(powder=os.path.join(setup.root_dir, f"powder/powder_max_r{setup.run:04}.npy"), + dist = geom_opt.opt_distance(powder=os.path.join(setup.root_dir, f"powder/r{setup.run:04}_max.npy"), center=task.center, - plot=os.path.join(taskdir, f'r{setup.run:04}.png')) + plot=os.path.join(taskdir, f'figs/r{setup.run:04}.png')) logger.info(f'Detector distance inferred from powder rings: {dist} mm') - temp_file = os.path.join(taskdir, 'temp.geom') - geom_file = os.path.join(taskdir, f'r{setup.run:04}.geom') - generate_geom_file(setup.exp, setup.run, setup.det_type, task.input_geom, temp_file, det_dist=dist) - modify_crystfel_header(temp_file, geom_file) - os.remove(temp_file) - logger.info(f'CrystFEL geom file saved with updated coffset value to: {geom_file}') + geom_in = fetch_latest(fnames=os.path.join(setup.root_dir, 'geom', 'r*.geom'), run=setup.run) + geom_temp = os.path.join(taskdir, 'temp.geom') + geom_out = os.path.join(taskdir, f'r{setup.run:04}.geom') + generate_geom_file(setup.exp, setup.run, setup.det_type, geom_in, geom_temp, det_dist=dist) + modify_crystfel_header(geom_temp, geom_out) + os.remove(geom_temp) + logger.info(f'CrystFEL geom file saved with updated coffset value to: {geom_out}') logger.debug('Done!') def find_peaks(config): diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index 59cdac32d..cdc0ae330 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -14,7 +14,6 @@ run_analysis: opt_distance: center: 960 960 - input_geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/176-end.data' find_peaks: tag: '' From 208241628ac8a8755a203e01ea5ab0e9bd800f72 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sun, 1 May 2022 07:46:43 -0700 Subject: [PATCH 12/14] using fetch_latest during find_peaks task --- scripts/tasks.py | 4 +++- tutorial/mfxlv4920.yaml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/tasks.py b/scripts/tasks.py index 8d57de34a..806aa1c13 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -92,13 +92,15 @@ def opt_distance(config): def find_peaks(config): from btx.processing.peak_finder import PeakFinder + from btx.misc.shortcuts import fetch_latest setup = config.setup task = config.find_peaks """ Perform adaptive peak finding on run. """ taskdir = os.path.join(setup.root_dir, 'index') os.makedirs(taskdir, exist_ok=True) + mask_file = fetch_latest(fnames=os.path.join(setup.root_dir, 'mask', 'r*.npy'), run=setup.run) pf = PeakFinder(exp=setup.exp, run=setup.run, det_type=setup.det_type, outdir=os.path.join(taskdir ,f"r{setup.run:04}"), - tag=task.tag, mask=task.mask, psana_mask=task.psana_mask, min_peaks=task.min_peaks, max_peaks=task.max_peaks, + tag=task.tag, mask=mask_file, psana_mask=task.psana_mask, min_peaks=task.min_peaks, max_peaks=task.max_peaks, npix_min=task.npix_min, npix_max=task.npix_max, amax_thr=task.amax_thr, atot_thr=task.atot_thr, son_min=task.son_min, peak_rank=task.peak_rank, r0=task.r0, dr=task.dr, nsigm=task.nsigm) logger.debug(f'Performing peak finding for run {setup.run} of {setup.exp}...') diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index cdc0ae330..e998bec14 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -1,7 +1,7 @@ setup: root_dir: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/' exp: 'mfxlv4920' - run: 175 + run: 234 det_type: 'Rayonix' fetch_mask: @@ -17,7 +17,6 @@ opt_distance: find_peaks: tag: '' - mask: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/rayonix_mask.npy' psana_mask: False min_peaks: 10 max_peaks: 2048 From f9a861c7155cce5457cba78f8c28e6459466b09f Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sun, 1 May 2022 08:03:18 -0700 Subject: [PATCH 13/14] updating indexing: tag nomenclature, fetch_latest for geom --- btx/processing/indexer.py | 10 ++++++---- scripts/tasks.py | 8 +++++--- tutorial/mfxlv4920.yaml | 7 +++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/btx/processing/indexer.py b/btx/processing/indexer.py index cb16b1aaa..7054a580a 100644 --- a/btx/processing/indexer.py +++ b/btx/processing/indexer.py @@ -6,7 +6,7 @@ class Indexer: """ Index cxi files using indexamajig: https://www.desy.de/~twhite/crystfel/manual-indexamajig.html """ - def __init__(self, exp, run, det_type, taskdir, geom, cell, int_rad='4,5,6', methods='mosflm', + def __init__(self, exp, run, det_type, tag, taskdir, geom, cell, int_rad='4,5,6', methods='mosflm', tolerance='5,5,5,1.5', no_revalidate=True, multi=True, profile=True): # general paramters @@ -23,7 +23,7 @@ def __init__(self, exp, run, det_type, taskdir, geom, cell, int_rad='4,5,6', met self.no_revalidate = no_revalidate # bool, skip validation step to omit iffy peaks self.multi = multi # bool, enable multi-lattice indexing self.profile = profile # bool, display timing data - self._retrieve_paths(taskdir) + self._retrieve_paths(taskdir, tag) self._parallel_logic() def _parallel_logic(self): @@ -39,7 +39,7 @@ def _parallel_logic(self): else: self.rank = 0 - def _retrieve_paths(self, taskdir): + def _retrieve_paths(self, taskdir, tag): """ Retrieve the paths for the input .lst and output .stream file consistent with the btx analysis directory structure. @@ -48,9 +48,11 @@ def _retrieve_paths(self, taskdir): ---------- taskdir : str path to output folder for indexing results + tag : str + filename extension suffix """ self.lst = os.path.join(taskdir ,f'r{self.run:04}/r{self.run:04}.lst') - self.stream = os.path.join(taskdir, f'r{self.run:04}.stream') + self.stream = os.path.join(taskdir, f'r{self.run:04}_{tag}.stream') if "TMP_EXE" in os.environ: self.tmp_exe = os.environ['TMP_EXE'] else: diff --git a/scripts/tasks.py b/scripts/tasks.py index 806aa1c13..7382d0d74 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -116,13 +116,15 @@ def find_peaks(config): def index(config): from btx.processing.indexer import Indexer + from btx.misc.shortcuts import fetch_latest setup = config.setup task = config.index """ Index run using indexamajig. """ taskdir = os.path.join(setup.root_dir, 'index') - indexer_obj = Indexer(exp=config.setup.exp, run=config.setup.run, det_type=config.setup.det_type, taskdir=taskdir, geom=task.geom, - cell=task.cell, int_rad=task.int_radius, methods=task.methods, tolerance=task.tolerance, - no_revalidate=task.no_revalidate, multi=task.multi, profile=task.profile) + geom_file = fetch_latest(fnames=os.path.join(setup.root_dir, 'geom', 'r*.geom'), run=setup.run) + indexer_obj = Indexer(exp=config.setup.exp, run=config.setup.run, det_type=config.setup.det_type, tag=task.tag, + taskdir=taskdir, geom=geom_file, cell=task.cell, int_rad=task.int_radius, methods=task.methods, + tolerance=task.tolerance, no_revalidate=task.no_revalidate, multi=task.multi, profile=task.profile) logger.debug(f'Generating indexing executable for run {setup.run} of {setup.exp}...') indexer_obj.write_exe() logger.info(f'Executable written to {indexer_obj.tmp_exe}') diff --git a/tutorial/mfxlv4920.yaml b/tutorial/mfxlv4920.yaml index e998bec14..ebd26ece8 100644 --- a/tutorial/mfxlv4920.yaml +++ b/tutorial/mfxlv4920.yaml @@ -1,7 +1,7 @@ setup: root_dir: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/' exp: 'mfxlv4920' - run: 234 + run: 235 det_type: 'Rayonix' fetch_mask: @@ -31,8 +31,7 @@ find_peaks: nsigm: 10.0 index: - tag: '' - geom: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/geom/r0175.geom' + tag: 'test' int_radius: '4,5,6' methods: 'mosflm' cell: '/cds/data/psdm/mfx/mfxlv4920/scratch/apeck/cco.cell' @@ -42,4 +41,4 @@ index: profile: True stream_analysis: - tag: '' + tag: 'test' From 17d1ebf147b6497e064d065680955f020f568b16 Mon Sep 17 00:00:00 2001 From: Ariana Peck Date: Sun, 1 May 2022 08:20:48 -0700 Subject: [PATCH 14/14] updated stream_analysis figure nomenclature --- scripts/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tasks.py b/scripts/tasks.py index 7382d0d74..a06f32b7e 100644 --- a/scripts/tasks.py +++ b/scripts/tasks.py @@ -135,11 +135,12 @@ def stream_analysis(config): task = config.stream_analysis """ Diagnostics including cell distribution and peakogram. """ taskdir = os.path.join(setup.root_dir, 'index') + os.makedirs(os.path.join(taskdir, 'figs'), exist_ok=True) stream_files = glob.glob(os.path.join(taskdir, f"r*{task.tag}.stream")) st = StreamInterface(input_files=stream_files, cell_only=False) if st.rank == 0: logger.debug(f'Read stream files: {stream_files}') - st.plot_cell_parameters(output=os.path.join(taskdir, f"{task.tag}_cell_distribution.png")) - st.plot_peakogram(output=os.path.join(taskdir, f"{task.tag}_peakogram.png")) + st.plot_cell_parameters(output=os.path.join(taskdir, f"figs/cell_{task.tag}.png")) + st.plot_peakogram(output=os.path.join(taskdir, f"figs/peakogram_{task.tag}.png")) logger.info(f'Peakogram and cell distribution generated for sample {task.tag}') logger.debug('Done!')