From 5cbcf3f03c62bba0552c3f65cb51c0537cfdf405 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Fri, 18 Jun 2021 15:06:26 +0200 Subject: [PATCH 1/8] split_exp runner revision --- .../SplitExp_package/SplitExp_script.py | 110 ++++++++++++++++++ shapepipe/modules/split_exp_runner.py | 91 +++------------ 2 files changed, 128 insertions(+), 73 deletions(-) create mode 100644 shapepipe/modules/SplitExp_package/SplitExp_script.py diff --git a/shapepipe/modules/SplitExp_package/SplitExp_script.py b/shapepipe/modules/SplitExp_package/SplitExp_script.py new file mode 100644 index 000000000..522103e31 --- /dev/null +++ b/shapepipe/modules/SplitExp_package/SplitExp_script.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""SPLIT EXP SCRIPT + +Class to split single-exposure multi-CCD mosaic images into single-exposure single-CCD +files, one HDU per CCD. + +:Author: Axel Guinot + +:Date: 2019, 2020 + +:Package: ShapePipe + +""" + +import numpy as np +import sip_tpv as stp +from astropy.wcs import WCS +from astropy.io import fits + +from shapepipe.pipeline import file_io as io + + +class SplitExposures(objec): + """Split Exposures initialisation + + Parameters + ---------- + input_file_list : list of strings + input file paths, typically image, weight, and flag + output_dir : string + output directory + file_number_string : string + input file identified + output_suffix : string + output file suffix + n_hdu : int + number of HDUs (= CCDs) of input files + """ + + def ___init___(self, input_file_list, output_dir, + file_number_string, output_suffix, n_hdu): + + self._input_file_list = input_file_list + self._output_dir = output_dir + self._file_number_string = file_number_string + self._output_suffix = output_suffix + self._n_hdu = n_hdu + + def process(self): + """Process + + Process the splitting of single-exposure images + """ + + for exp_path, output_suffix in zip(self.input_file_list, self.output_suffix): + + transf_int = 'flag' in exp_suffix + transf_coord = 'image' in exp_suffix + save_header = 'image' in exp_suffix + + self.create_hdus(exp_path, output_suffix, transf_coord, transf_int, save_header) + + def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): + """ Create HDUs + + Split a single exposures CCDs into separate files. + + Parameters + ---------- + exp_path : str + Path to the single exposure + output_sufix : str + Suffix for the output file + transf_coord : bool + Transform the WCS (pv to sip) if True + transf_int : bool + Set data types to int if True + save_header : bool + Save WCS information if True + """ + + header_file = np.zeros(self.n_hdu, dtype='O') + + for i in range(1, self.n_hdu+1): + + h = fits.getheader(exp_path, i) + if transf_coord: + stp.pv_to_sip(h) + + d = fits.getdata(exp_path, i) + if transf_int: + d = d.astype(np.int16) + + file_name = f'{self.output_dir}/{output_suffix}{self.file_number_string}-{str(i-1)}.fits' + + new_file = io.FITSCatalog(file_name, + open_mode=io.BaseCatalog.OpenMode.ReadWrite) + new_file.save_as_fits(data=d, image=True, image_header=h) + + if save_header: + try: + w = WCS(h) + except: + print('WCS error for file {}'.format(exp_path)) + raise + header_file[i-1] = {'WCS': w, 'header': h.tostring()} + + if save_header: + file_name = f'{self.output_dir}/headers{self.file_number_string}/npy' + np.save(file_name, header_file) diff --git a/shapepipe/modules/split_exp_runner.py b/shapepipe/modules/split_exp_runner.py index 389a121fd..8eebca938 100644 --- a/shapepipe/modules/split_exp_runner.py +++ b/shapepipe/modules/split_exp_runner.py @@ -2,93 +2,38 @@ """SPLIT EXP RUNNER -This module split the different CCD's hdu of a single exposure into separate -files. +This module splits the different CCDs (= hdus in FITS files) of a +single exposure into separate files. :Author: Axel Guinot -""" - - -import numpy as np -import sip_tpv as stp -from astropy.wcs import WCS -from astropy.io import fits - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.pipeline import file_io as io - - -def create_hdus(exp_path, output_dir, output_name, output_sufix, n_hdu=40, - transf_coord=True, transf_int=False, save_header=False): - """ Create HDUs - - Split a single exposures CCDs into separate files. +:Date: 2019, 2020 - exp_path : str - Path to the single exp. - output_dir : str - Path to the output directory. - output_sufix : str - Suffix for the output file. - n_hdu : int - Number of HDUs (i.e. : number of CCDs). - transf_coord : bool - If True will transform the WCS (pv to sip). - transf_int : bool - If True will set datas to int. - save_header : bool - If True will save WCS information +:Package: ShapePipe - """ - - header_file = np.zeros(n_hdu, dtype='O') - - for i in range(1, n_hdu+1): - - h = fits.getheader(exp_path, i) - if transf_coord: - stp.pv_to_sip(h) - - d = fits.getdata(exp_path, i) - if transf_int: - d = d.astype(np.int16) - - file_name = (output_dir + '/' + output_sufix + output_name + - '-' + str(i-1) + '.fits') - new_file = io.FITSCatalog(file_name, - open_mode=io.BaseCatalog.OpenMode.ReadWrite) - new_file.save_as_fits(data=d, image=True, image_header=h) +""" - if save_header: - try: - w = WCS(h) - except: - print('WCS error for file {}'.format(exp_path)) - raise - header_file[i-1] = {'WCS': w, 'header': h.tostring()} - if save_header: - file_name = output_dir + '/' + 'headers' + output_name + '.npy' - np.save(file_name, header_file) +from shapepipe.modules.module_decorator import module_runner -@module_runner(version='1.0', file_pattern=['image', 'weight', 'flag'], - file_ext=['.fz', '.fz', '.fz'], - depends=['numpy', 'astropy', 'sip_tpv']) +@module_runner( + version='1.0', + input_module='get_images_runner', + file_pattern=['image', 'weight', 'flag'], + file_ext=['.fz', '.fz', '.fz'], + depends=['numpy', 'astropy', 'sip_tpv'], + run_method='parallel' +) def split_exp_runner(input_file_list, run_dirs, file_number_string, config, w_log): - file_suffix = config.getlist("SPLIT_EXP_RUNNER", "OUTPUT_SUFFIX") + output_suffix = config.getlist("SPLIT_EXP_RUNNER", "OUTPUT_SUFFIX") n_hdu = config.getint("SPLIT_EXP_RUNNER", "N_HDU") - for exp_path, exp_suffix in zip(input_file_list, file_suffix): - - transf_int = 'flag' in exp_suffix - transf_coord = 'image' in exp_suffix - save_header = 'image' in exp_suffix + inst = split.SplitExposures(input_file_list, run_dirs['output'], + file_number_string, output_suffix, n_hdu) - create_hdus(exp_path, run_dirs['output'], file_number_string, - exp_suffix, n_hdu, transf_coord, transf_int, save_header) + inst.process() return None, None From b4bcbf384240e33f07452da909e8c23850bd0446 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 18 Jun 2021 17:21:06 +0200 Subject: [PATCH 2/8] split exp runner bug fixes --- .../SplitExp_package/SplitExp_script.py | 26 +++++++++---------- shapepipe/modules/split_exp_runner.py | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/shapepipe/modules/SplitExp_package/SplitExp_script.py b/shapepipe/modules/SplitExp_package/SplitExp_script.py index 522103e31..9884ebb1d 100644 --- a/shapepipe/modules/SplitExp_package/SplitExp_script.py +++ b/shapepipe/modules/SplitExp_package/SplitExp_script.py @@ -20,7 +20,7 @@ from shapepipe.pipeline import file_io as io -class SplitExposures(objec): +class SplitExposures(object): """Split Exposures initialisation Parameters @@ -37,8 +37,8 @@ class SplitExposures(objec): number of HDUs (= CCDs) of input files """ - def ___init___(self, input_file_list, output_dir, - file_number_string, output_suffix, n_hdu): + def __init__(self, input_file_list, output_dir, + file_number_string, output_suffix, n_hdu): self._input_file_list = input_file_list self._output_dir = output_dir @@ -52,15 +52,15 @@ def process(self): Process the splitting of single-exposure images """ - for exp_path, output_suffix in zip(self.input_file_list, self.output_suffix): + for exp_path, output_suffix in zip(self._input_file_list, self._output_suffix): - transf_int = 'flag' in exp_suffix - transf_coord = 'image' in exp_suffix - save_header = 'image' in exp_suffix + transf_int = 'flag' in output_suffix + transf_coord = 'image' in output_suffix + save_header = 'image' in output_suffix self.create_hdus(exp_path, output_suffix, transf_coord, transf_int, save_header) - def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): + def create_hdus(self, exp_path, output_suffix, transf_coord, transf_int, save_header): """ Create HDUs Split a single exposures CCDs into separate files. @@ -69,7 +69,7 @@ def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): ---------- exp_path : str Path to the single exposure - output_sufix : str + output_suffix : str Suffix for the output file transf_coord : bool Transform the WCS (pv to sip) if True @@ -79,9 +79,9 @@ def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): Save WCS information if True """ - header_file = np.zeros(self.n_hdu, dtype='O') + header_file = np.zeros(self._n_hdu, dtype='O') - for i in range(1, self.n_hdu+1): + for i in range(1, self._n_hdu+1): h = fits.getheader(exp_path, i) if transf_coord: @@ -91,7 +91,7 @@ def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): if transf_int: d = d.astype(np.int16) - file_name = f'{self.output_dir}/{output_suffix}{self.file_number_string}-{str(i-1)}.fits' + file_name = f'{self._output_dir}/{output_suffix}{self._file_number_string}-{str(i-1)}.fits' new_file = io.FITSCatalog(file_name, open_mode=io.BaseCatalog.OpenMode.ReadWrite) @@ -106,5 +106,5 @@ def create_hdus(exp_path, output_sufix, transf_coord, transf_int, save_header): header_file[i-1] = {'WCS': w, 'header': h.tostring()} if save_header: - file_name = f'{self.output_dir}/headers{self.file_number_string}/npy' + file_name = f'{self._output_dir}/headers{self._file_number_string}.npy' np.save(file_name, header_file) diff --git a/shapepipe/modules/split_exp_runner.py b/shapepipe/modules/split_exp_runner.py index 8eebca938..18bf8608a 100644 --- a/shapepipe/modules/split_exp_runner.py +++ b/shapepipe/modules/split_exp_runner.py @@ -15,6 +15,7 @@ from shapepipe.modules.module_decorator import module_runner +from shapepipe.modules.SplitExp_package import SplitExp_script as split @module_runner( From c7ad1290ba33507143ec60ba5a50dfea7d66958a Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:16:52 +0200 Subject: [PATCH 3/8] runner to new syntax --- shapepipe/modules/split_exp_runner.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/shapepipe/modules/split_exp_runner.py b/shapepipe/modules/split_exp_runner.py index 18bf8608a..7bbfa6940 100644 --- a/shapepipe/modules/split_exp_runner.py +++ b/shapepipe/modules/split_exp_runner.py @@ -26,14 +26,23 @@ depends=['numpy', 'astropy', 'sip_tpv'], run_method='parallel' ) -def split_exp_runner(input_file_list, run_dirs, file_number_string, - config, w_log): +def split_exp_runner( + input_file_list, + run_dirs, + file_number_string, + config, + w_log): output_suffix = config.getlist("SPLIT_EXP_RUNNER", "OUTPUT_SUFFIX") n_hdu = config.getint("SPLIT_EXP_RUNNER", "N_HDU") - inst = split.SplitExposures(input_file_list, run_dirs['output'], - file_number_string, output_suffix, n_hdu) + inst = split.SplitExposures( + input_file_list, + run_dirs['output'], + file_number_string, + output_suffix, + n_hdu + ) inst.process() From 43744572fb4828acee793cba1c5e75c2e5d74484 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:23:29 +0200 Subject: [PATCH 4/8] Split_Exp package to new syntax --- .../Split_Exp_script.py} | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) rename shapepipe/modules/{SplitExp_package/SplitExp_script.py => Split_Exp_package/Split_Exp_script.py} (69%) diff --git a/shapepipe/modules/SplitExp_package/SplitExp_script.py b/shapepipe/modules/Split_Exp_package/Split_Exp_script.py similarity index 69% rename from shapepipe/modules/SplitExp_package/SplitExp_script.py rename to shapepipe/modules/Split_Exp_package/Split_Exp_script.py index 9884ebb1d..3f66e5a4f 100644 --- a/shapepipe/modules/SplitExp_package/SplitExp_script.py +++ b/shapepipe/modules/Split_Exp_package/Split_Exp_script.py @@ -37,8 +37,14 @@ class SplitExposures(object): number of HDUs (= CCDs) of input files """ - def __init__(self, input_file_list, output_dir, - file_number_string, output_suffix, n_hdu): + def __init__( + self, + input_file_list, + output_dir, + file_number_string, + output_suffix, + n_hdu + ): self._input_file_list = input_file_list self._output_dir = output_dir @@ -52,15 +58,31 @@ def process(self): Process the splitting of single-exposure images """ - for exp_path, output_suffix in zip(self._input_file_list, self._output_suffix): + for exp_path, output_suffix in zip( + self._input_file_list, + self._output_suffix + ): transf_int = 'flag' in output_suffix transf_coord = 'image' in output_suffix save_header = 'image' in output_suffix - self.create_hdus(exp_path, output_suffix, transf_coord, transf_int, save_header) - - def create_hdus(self, exp_path, output_suffix, transf_coord, transf_int, save_header): + self.create_hdus( + exp_path, + output_suffix, + transf_coord, + transf_int, + save_header + ) + + def create_hdus( + self, + exp_path, + output_suffix, + transf_coord, + transf_int, + save_header + ): """ Create HDUs Split a single exposures CCDs into separate files. @@ -91,20 +113,27 @@ def create_hdus(self, exp_path, output_suffix, transf_coord, transf_int, save_he if transf_int: d = d.astype(np.int16) - file_name = f'{self._output_dir}/{output_suffix}{self._file_number_string}-{str(i-1)}.fits' + file_name = ( + f'{self._output_dir}/{output_suffix}' + + '{self._file_number_string}-{str(i-1)}.fits' + ) - new_file = io.FITSCatalog(file_name, - open_mode=io.BaseCatalog.OpenMode.ReadWrite) + new_file = io.FITSCatalog( + file_name, + open_mode=io.BaseCatalog.OpenMode.ReadWrite + ) new_file.save_as_fits(data=d, image=True, image_header=h) if save_header: try: w = WCS(h) except: - print('WCS error for file {}'.format(exp_path)) + print(f'WCS error for file {exp_path}') raise header_file[i-1] = {'WCS': w, 'header': h.tostring()} if save_header: - file_name = f'{self._output_dir}/headers{self._file_number_string}.npy' + file_name = ( + f'{self._output_dir}/headers{self._file_number_string}.npy' + ) np.save(file_name, header_file) From 51dbbaa8457357408874fedec7a9a78f4ef79a5e Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:25:29 +0200 Subject: [PATCH 5/8] passed test --- shapepipe/modules/split_exp_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shapepipe/modules/split_exp_runner.py b/shapepipe/modules/split_exp_runner.py index 7bbfa6940..c05d94abd 100644 --- a/shapepipe/modules/split_exp_runner.py +++ b/shapepipe/modules/split_exp_runner.py @@ -31,7 +31,8 @@ def split_exp_runner( run_dirs, file_number_string, config, - w_log): + w_log +): output_suffix = config.getlist("SPLIT_EXP_RUNNER", "OUTPUT_SUFFIX") n_hdu = config.getint("SPLIT_EXP_RUNNER", "N_HDU") From 7923484d2da3af70bc9c1913e4ef068d987e793f Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:30:49 +0200 Subject: [PATCH 6/8] Rename Split_Exp_script.py to split_exp_script.py renamed file --- .../{Split_Exp_script.py => split_exp_script.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shapepipe/modules/Split_Exp_package/{Split_Exp_script.py => split_exp_script.py} (100%) diff --git a/shapepipe/modules/Split_Exp_package/Split_Exp_script.py b/shapepipe/modules/Split_Exp_package/split_exp_script.py similarity index 100% rename from shapepipe/modules/Split_Exp_package/Split_Exp_script.py rename to shapepipe/modules/Split_Exp_package/split_exp_script.py From 5a88c6b7e97c60969395ca64c4b815d7064f6a10 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:32:41 +0200 Subject: [PATCH 7/8] split exp runner and package consistent file names --- .../{Split_Exp_package => split_exp_package}/split_exp_script.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shapepipe/modules/{Split_Exp_package => split_exp_package}/split_exp_script.py (100%) diff --git a/shapepipe/modules/Split_Exp_package/split_exp_script.py b/shapepipe/modules/split_exp_package/split_exp_script.py similarity index 100% rename from shapepipe/modules/Split_Exp_package/split_exp_script.py rename to shapepipe/modules/split_exp_package/split_exp_script.py From a7bbae4073fcc5102f8580a100989576bb9b9591 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Thu, 15 Jul 2021 16:34:05 +0200 Subject: [PATCH 8/8] fixed import bug --- shapepipe/modules/split_exp_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapepipe/modules/split_exp_runner.py b/shapepipe/modules/split_exp_runner.py index c05d94abd..5c1a2df5e 100644 --- a/shapepipe/modules/split_exp_runner.py +++ b/shapepipe/modules/split_exp_runner.py @@ -15,7 +15,7 @@ from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.SplitExp_package import SplitExp_script as split +from shapepipe.modules.split_exp_package import split_exp_script as split @module_runner(