diff --git a/docs/change_history.rst b/docs/change_history.rst index 276172c5..d0f0a027 100644 --- a/docs/change_history.rst +++ b/docs/change_history.rst @@ -3,9 +3,15 @@ Change History .. _v1.3.0: -V1.3.0 Unreleased +V1.3.0 06-03-2020 ^^^^^^^^^^^^^^^^^ +- Made it compatible with Astropy 4.0 +- All versions are free except for Pandas [#314] +- `wavelength.WavelengthCalibration.__call__` can now return a json output. +- `core.setup_logging` can now create a generic logger (same format). +- Modified how master bias are named. +- Removed bias overscan and trimming correction on master bias creation. - Bugs Fixed: + `--max-targets` was not being used, missed connection in `MainApp`. @@ -20,6 +26,7 @@ V1.3.0 Unreleased points, this allows to re use other master loggers. - Changed `--background-threshold` to multiply by detection limit instead of background level +- Created standard JSON output for :class:`~wavelength.WavelengthCalibration`. .. _v1.2.1: diff --git a/goodman_pipeline/core/core.py b/goodman_pipeline/core/core.py index 5f37fc95..2cadeaa7 100644 --- a/goodman_pipeline/core/core.py +++ b/goodman_pipeline/core/core.py @@ -2,24 +2,21 @@ unicode_literals) import calendar +import ccdproc import collections import datetime import glob import logging import math +import numpy as np import os +import pandas import re -import shutil +import scipy import subprocess import sys import time -from threading import Timer -import pickle -import ccdproc -import numpy as np -import pandas -import scipy from astroplan import Observer from astropy import units as u from astropy.io import fits @@ -30,9 +27,9 @@ from astropy.time import Time from astroscrappy import detect_cosmics from ccdproc import CCDData, ImageFileCollection -# matplotlib.use('Qt5Agg') from matplotlib import pyplot as plt from scipy import signal +from threading import Timer from . import check_version @@ -41,9 +38,6 @@ log = logging.getLogger(__name__) - - - def astroscrappy_lacosmic(ccd, red_path=None, save_mask=False): mask, ccd.data = detect_cosmics(ccd.data) @@ -424,7 +418,7 @@ def create_master_flats(flat_files, trim_section, master_bias_name, new_master_flat_name, - saturation, + saturation_threshold, ignore_bias=False): """Creates master flats @@ -450,7 +444,7 @@ def create_master_flats(flat_files, the full path as `raw_path` + `basename`. new_master_flat_name (str): Name of the file to save new master flat. Can be absolute path or not. - saturation (int): Saturation threshold, defines the percentage of + saturation_threshold (int): Saturation threshold, defines the percentage of pixels above saturation level allowed for flat field images. ignore_bias (bool): Flag to create master bias without master bias. @@ -508,11 +502,11 @@ def create_master_flats(flat_files, 'Master bias image') else: - log.error('Unknown observation technique: ' + technique) + log.warning('Ignoring bias on request') if is_file_saturated(ccd=ccd, - threshold=saturation): + threshold=saturation_threshold): log.warning('Removing saturated image {:s}. ' - 'Use --saturation to change saturation ' + 'Use --saturation_threshold to change saturation_threshold ' 'level'.format(flat_file)) continue else: @@ -541,7 +535,7 @@ def create_master_flats(flat_files, return master_flat, master_flat_name else: log.error('Empty flat list. Check that they do not exceed the ' - 'saturation limit.') + 'saturation_threshold limit.') return None, None @@ -1074,9 +1068,9 @@ def extraction(ccd, 2D spectrum target_trace (object): Instance of astropy.modeling.Model, a low order polynomial that defines the trace of the spectrum in the ccd object. - spatial_profile (Model): Instance of astropy.modeling.Model, a Gaussian - model previously fitted to the spatial profile of the 2D spectrum - contained in the ccd object. + spatial_profile (Model): Instance of :class:`~astropy.modeling.Model`, + a Gaussian model previously fitted to the spatial profile of the 2D + spectrum contained in the ccd object. extraction_name (str): Extraction type, can be `fractional` or `optimal` though the optimal extraction is not implemented yet. @@ -1991,9 +1985,9 @@ def image_trim(ccd, trim_section, trim_type='trimsec', add_keyword=False): 'Slit trim section, slit illuminated ' 'area only.') else: - log.warning('Unrecognized trim type') + log.warning('Unrecognized trim type: {}'.format(trim_type)) ccd.header['GSP_TRIM'] = (trim_section, - 'Image trimmed by unreckognized method: ' + 'Image trimmed by unrecognized method: ' '{:s}'.format(trim_type)) else: log.info("{:s} trim section is not " @@ -2038,13 +2032,13 @@ def interpolate(spectrum, interpolation_size): def is_file_saturated(ccd, threshold): """Detects a saturated image - It counts the number of pixels above the saturation level, then finds + It counts the number of pixels above the saturation_threshold level, then finds which percentage they represents and if it is above the threshold it will return True. The percentage threshold can be set using the command - line argument ``--saturation``. + line argument ``--saturation_threshold``. Args: - ccd (CCDData): Image to be tested for saturation + ccd (CCDData): Image to be tested for saturation_threshold threshold (float): Percentage of saturated pixels allowed. Default 1. Returns: @@ -2066,7 +2060,7 @@ def is_file_saturated(ccd, threshold): if saturated_percent >= float(threshold): log.warning( "The current image has more than {:.2f} percent " - "of pixels above saturation level".format(float(threshold))) + "of pixels above saturation_threshold level".format(float(threshold))) return True else: return False @@ -3912,18 +3906,18 @@ class SaturationValues(object): """ def __init__(self, ccd=None): - """Defines a :class:`~pandas.DataFrame` with saturation information + """Defines a :class:`~pandas.DataFrame` with saturation_threshold information - Both, Red and Blue cameras have tabulated saturation values depending + Both, Red and Blue cameras have tabulated saturation_threshold values depending on the readout configurations. It defines a :class:`~pandas.DataFrame` object. Notes: For the purposes of this documentation *50% full well* is the same - as ``saturation level`` though they are not the same thing. + as ``saturation_threshold level`` though they are not the same thing. Args: - ccd (CCDData): Image to be tested for saturation + ccd (CCDData): Image to be tested for saturation_threshold """ self.log = logging.getLogger(__name__) @@ -3964,7 +3958,7 @@ def saturation_value(self): """Saturation value in counts In fact the value it returns is the 50% of full potential well, - Some configurations reach digital saturation before 50% of full + Some configurations reach digital saturation_threshold before 50% of full potential well, they are specified in the last column: ``saturates_before``. @@ -3979,13 +3973,13 @@ def saturation_value(self): return self.__saturation def get_saturation_value(self, ccd): - """Defines the saturation level + """Defines the saturation_threshold level Args: - ccd (CCDData): Image to be tested for saturation + ccd (CCDData): Image to be tested for saturation_threshold Returns: - The saturation value or None + The saturation_threshold value or None """ hfw = self._sdf.half_full_well[ @@ -3994,12 +3988,12 @@ def get_saturation_value(self, ccd): (self._sdf.read_noise == ccd.header['RDNOISE'])] if hfw.empty: - self.log.critical('Unable to obtain saturation level') + self.log.critical('Unable to obtain saturation_threshold level') self.__saturation = None return None else: self.__saturation = float(hfw.to_string(index=False)) - self.log.debug("Set saturation level as {:.0f}".format( + self.log.debug("Set saturation_threshold level as {:.0f}".format( self.__saturation)) return self.__saturation diff --git a/goodman_pipeline/core/tests/test_core.py b/goodman_pipeline/core/tests/test_core.py index c29e1038..c2835117 100644 --- a/goodman_pipeline/core/tests/test_core.py +++ b/goodman_pipeline/core/tests/test_core.py @@ -609,7 +609,7 @@ def test_create_master_flats_no_bias(self): trim_section=self.trim_section, master_bias_name='master_bias.fits', new_master_flat_name=self.master_flat_name, - saturation=1, + saturation_threshold=1, ignore_bias=True) self.assertEqual(self.master_flat_name, os.path.basename(name)) @@ -629,7 +629,7 @@ def test_create_master_flats_with_bias(self): 'master_bias.fits'), new_master_flat_name=os.path.join(self.reduced_data, self.master_flat_name), - saturation=1, + saturation_threshold=1, ignore_bias=False) self.assertEqual(self.master_flat_name, os.path.basename(name)) @@ -656,7 +656,7 @@ def test_create_master_flats_saturated_flats(self): trim_section=self.trim_section, master_bias_name='master_bias.fits', new_master_flat_name=self.master_flat_name, - saturation=1, + saturation_threshold=1, ignore_bias=False) self.assertNotEqual(self.flat_files[0], master.header['GSP_IC01']) @@ -671,7 +671,7 @@ def test_create_master_flats_empty_list(self): trim_section=self.trim_section, master_bias_name='master_bias.fits', new_master_flat_name=self.master_flat_name, - saturation=1, + saturation_threshold=1, ignore_bias=False) self.assertIsNone(master) self.assertIsNone(name) diff --git a/goodman_pipeline/images/goodman_ccd.py b/goodman_pipeline/images/goodman_ccd.py index 61da2c4e..d102af44 100755 --- a/goodman_pipeline/images/goodman_ccd.py +++ b/goodman_pipeline/images/goodman_ccd.py @@ -131,12 +131,12 @@ def get_args(arguments=None): default='./RED', help="Path to reduced data.") - parser.add_argument('--saturation', + parser.add_argument('--saturation_threshold', action='store', default=1., dest='saturation_threshold', metavar='', - help="Maximum percent of pixels above saturation " + help="Maximum percent of pixels above saturation_threshold " "threshold. Default 1 percent.") parser.add_argument('--version', diff --git a/goodman_pipeline/images/image_processor.py b/goodman_pipeline/images/image_processor.py index caf3e700..5fb47060 100644 --- a/goodman_pipeline/images/image_processor.py +++ b/goodman_pipeline/images/image_processor.py @@ -142,7 +142,7 @@ def __call__(self): trim_section=self.trim_section, master_bias_name=self.master_bias_name, new_master_flat_name=master_flat_name, - saturation=self.args.saturation_threshold, + saturation_threshold=self.args.saturation_threshold, ignore_bias=self.args.ignore_bias) else: log.debug('Process Data Group') @@ -219,7 +219,7 @@ def process_spectroscopy_science(self, science_group, save_all=False): trim_section=self.trim_section, master_bias_name=self.master_bias_name, new_master_flat_name=master_flat_name, - saturation=self.args.saturation_threshold, + saturation_threshold=self.args.saturation_threshold, ignore_bias=self.args.ignore_bias) elif self.args.ignore_flats: log.warning('Ignoring creation of Master Flat by request.') @@ -494,7 +494,7 @@ def process_spectroscopy_science(self, science_group, save_all=False): trim_section=self.trim_section, master_bias_name=self.master_bias_name, new_master_flat_name=master_flat_name, - saturation=self.args.saturation_threshold) + saturation_threshold=self.args.saturation_threshold) else: log.error('There is no valid datatype in this group') diff --git a/goodman_pipeline/images/tests/test_image_processor.py b/goodman_pipeline/images/tests/test_image_processor.py index 3d923936..06d79f20 100644 --- a/goodman_pipeline/images/tests/test_image_processor.py +++ b/goodman_pipeline/images/tests/test_image_processor.py @@ -14,7 +14,7 @@ class ImageProcessorTest(TestCase): def setUp(self): - arguments = ['--saturation', '1'] + arguments = ['--saturation_threshold', '1'] args = get_args(arguments=arguments) data_container = NightDataContainer(path='/fake', instrument='Red', diff --git a/goodman_pipeline/spectroscopy/redspec.py b/goodman_pipeline/spectroscopy/redspec.py index b171e271..46776cb9 100755 --- a/goodman_pipeline/spectroscopy/redspec.py +++ b/goodman_pipeline/spectroscopy/redspec.py @@ -527,7 +527,7 @@ def _run(self, self.log.error(error) -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover MAIN_APP = MainApp() try: MAIN_APP() diff --git a/goodman_pipeline/spectroscopy/tests/test_wavelength.py b/goodman_pipeline/spectroscopy/tests/test_wavelength.py index 89694a83..5f7d73eb 100644 --- a/goodman_pipeline/spectroscopy/tests/test_wavelength.py +++ b/goodman_pipeline/spectroscopy/tests/test_wavelength.py @@ -2,6 +2,7 @@ import numpy as np import os +import re from astropy.convolution import convolve, Gaussian1DKernel, Box1DKernel from astropy.io import fits @@ -17,6 +18,7 @@ class WavelengthCalibrationTests(TestCase): def setUp(self): + self.file_list = [] argument_list = ['--data-path', os.getcwd(), '--proc-path', os.getcwd(), '--search-pattern', 'cfzsto', @@ -35,9 +37,114 @@ def setUp(self): self.ccd.header.set('SLIT', value='1.0_LONG_SLIT', comment="slit [arcsec]") + self.ccd.header.set('GSP_FNAM', + value='some_name.fits', + comment='Name of the current file') + self.ccd.header.set('OBSTYPE', + value='SPECTRUM', + comment='Obstype') + self.ccd.header.set('OBJECT', + value='An X Object', + comment='Some random object name') + self.ccd.header.set('GSP_FLAT', + value='some_flat_file.fits', + comment='The name of the flat') + self.ccd.header.set('CCDSUM', + value='1 1', + comment='Binning') + self.ccd.header.set('WAVMODE', + value='400 M1', + comment='wavmode') + self.lamp = self.ccd.copy() + self.lamp.header.set('OBSTYPE', + value='COMP', + comment='Comparison lamp obstype') + self.lamp.header.set('OBJECT', value='HgArNe') + def tearDown(self): + for _file in self.file_list: + if os.path.isfile(_file): + os.unlink(_file) @skip - def test_automatic_wavelength_solution(self): - pass \ No newline at end of file + def test__automatic_wavelength_solution(self): + pass + + def test__save_wavelength_calibrated(self): + self.wc.sci_target_file = 'target_sci_file.fits' + fname = self.wc._save_wavelength_calibrated(ccd=self.lamp, + original_filename='file_name.fits', + save_data_to=os.getcwd(), + lamp=True) + self.file_list.append(fname) + self.assertEqual(fname, os.path.join(os.getcwd(), 'wfile_name.fits')) + + fname = self.wc._save_wavelength_calibrated(ccd=self.lamp, + index=1, + original_filename='file_name.fits', + save_data_to=os.getcwd()) + self.file_list.append(fname) + self.assertEqual(fname, os.path.join(os.getcwd(), 'wfile_name_ws_1.fits')) + + def test__save_science_data(self): + wavelength_solution = models.Chebyshev1D(degree=3) + wavelength_solution.c0.value = 4419.161693945127 + wavelength_solution.c1.value = 1.321103785944705 + wavelength_solution.c2.value = -2.9766005683232e-06 + wavelength_solution.c3.value = -4.864180906701e-10 + fname = self.wc._save_science_data( + ccd=self.ccd, + wavelength_solution=wavelength_solution, + save_to=os.getcwd(), + index=None, + plot_results=False, + save_plots=False, + plots=False) + self.file_list.append(fname) + + fname_2 = self.wc._save_science_data( + ccd=self.ccd, + wavelength_solution=wavelength_solution, + save_to=os.getcwd(), + index=1, + plot_results=False, + save_plots=False, + plots=False) + self.file_list.append(fname_2) + expected_name = os.getcwd() + '/w' + re.sub('.fits', '', os.path.basename(self.ccd.header['GSP_FNAM'])) + "_ws_{:d}".format(1) + ".fits" + + self.assertEqual(fname, os.getcwd() + '/w' + os.path.basename(self.ccd.header['GSP_FNAM'])) + self.assertEqual(fname_2, expected_name) + + def test___call___method_wrong_ccd(self): + self.assertRaises(AssertionError, self.wc, [], [], '', '') + + def test___call___method_wrong_comp_list(self): + self.assertRaises(AssertionError, self.wc, self.ccd, 'comp_list', '', '') + + def test___call___method_no_comparison_lamps(self): + json_output = self.wc(ccd=self.ccd, + comp_list=[], + save_data_to='', + reference_data='', + json_output=True) + + self.assertEqual(json_output['error'], 'Unable to process without reference lamps') + self.assertEqual(json_output['warning'], '') + self.assertEqual(json_output['wavelength_solution'], []) + + def test___call___method_one_comparison_lamps(self): + json_output = self.wc(ccd=self.ccd, + comp_list=[self.lamp], + save_data_to='', + reference_data='goodman_pipeline/data/ref_comp', + json_output=True) + + print(json_output) + + self.assertEqual(json_output['error'], 'Unable to obtain wavelength solution') + self.assertEqual(json_output['warning'], '') + self.assertEqual(json_output['wavelength_solution'], []) + + diff --git a/goodman_pipeline/spectroscopy/wavelength.py b/goodman_pipeline/spectroscopy/wavelength.py index e4ba3f30..3edfa38a 100644 --- a/goodman_pipeline/spectroscopy/wavelength.py +++ b/goodman_pipeline/spectroscopy/wavelength.py @@ -82,6 +82,8 @@ def __init__(self): self.cross_corr_tolerance = 5 self.reference_data_dir = None self.reference_data = None + self.calibration_lamp = '' + self.wcal_lamp_file = '' # Instrument configuration and spectral characteristics self.serial_binning = None @@ -139,6 +141,10 @@ def __call__(self, assert isinstance(ccd, CCDData) assert isinstance(comp_list, list) + json_payload = {'wavelength_solution': [], + 'warning': '', + 'error': ''} + if os.path.isdir(reference_data): if self.reference_data_dir != reference_data: self.reference_data_dir = reference_data @@ -158,7 +164,10 @@ def __call__(self, "".format(self.sci_target_file)) log.error("Ending processing of {}".format(self.sci_target_file)) if json_output: - return {'error': 'Unable to process without reference lamps'} + json_payload['error'] ='Unable to process without reference lamps' + return json_payload + else: + return else: wavelength_solutions = [] reference_lamp_names = [] @@ -252,8 +261,9 @@ def __call__(self, 'reference_lamp': self.wcal_lamp_file}) if json_output: - return {'warning': warning_message, - 'wavelength_solution': all_solution_info} + json_payload['warning'] = warning_message + json_payload['wavelength_solution'] = all_solution_info + return json_payload elif len(wavelength_solutions) == 1: self.wsolution = wavelength_solutions[0] @@ -269,17 +279,19 @@ def __call__(self, index=object_number, plots=plots) if json_output: - return { - 'wavelength_solution': [ - {'solution_info': {'rms_error': "{:.4f}".format(self.rms_error), - 'npoints': "{:d}".format(self.n_points), - 'nrjections': "{:d}".format(self.n_rejections)}, - 'file_name': saved_file_name, - 'reference_lamp': self.wcal_lamp_file}]} + json_payload['wavelength_solution'] = [ + {'solution_info': {'rms_error': "{:.4f}".format(self.rms_error), + 'npoints': "{:d}".format(self.n_points), + 'nrjections': "{:d}".format(self.n_rejections)}, + 'file_name': saved_file_name, + 'reference_lamp': self.wcal_lamp_file}] + + return json_payload else: log.error("No wavelength solution.") if json_output: - return {'error': "no wavelength solution obtained"} + json_payload['error'] = "Unable to obtain wavelength solution" + return json_payload def _automatic_wavelength_solution(self, save_data_to, @@ -451,7 +463,7 @@ def _automatic_wavelength_solution(self, # correlation results clipped_values = sigma_clip(correlation_values, sigma=3, - iters=1, + maxiters=1, cenfunc=np.ma.median) # print(clipped_values) @@ -487,7 +499,7 @@ def _automatic_wavelength_solution(self, clipped_differences = sigma_clip(wavelength_differences, sigma=2, - iters=3, + maxiters=3, cenfunc=np.ma.median) if np.ma.is_masked(clipped_differences): @@ -625,7 +637,28 @@ def _save_science_data(self, plot_results=False, save_plots=False, plots=False): - """Save science data""" + """Save wavelength calibrated data + + The spectrum is linearized, then the linear solution is recorded in the + ccd's header and finally it calls the method + :func:`~wavelength.WavelengthCalibration._save_wavelength_calibrated` + which performs the actual saving to a file. + + Args: + ccd (CCDData): Instance of :class:`~astropy.nddata.CCDData` with a + 1D spectrum. + wavelength_solution (object): A :class:`~astropy.modeling.Model` + save_to (str): Path to save location + index (int): If there are more than one target, they are identified + by this index. + plot_results (bool): Whether to show plots or not. + save_plots (bool): Whether to save plots to files. + plots + + Returns: + File name of saved file. + + """ ccd = ccd.copy() linear_x_axis, ccd.data = linearize_spectrum( data=ccd.data, @@ -722,7 +755,7 @@ def _save_wavelength_calibrated(self, else: f_end = '_ws_{:d}.fits'.format(index) - new_filename = os.path.join(save_data_to, + file_full_path = os.path.join(save_data_to, output_prefix + original_filename.replace('.fits', f_end)) @@ -730,7 +763,7 @@ def _save_wavelength_calibrated(self, log.info('Wavelength-calibrated {:s} file saved to: ' '{:s} for science file {:s}' ''.format(ccd.header['OBSTYPE'], - os.path.basename(new_filename), + os.path.basename(file_full_path), self.sci_target_file)) ccd.header.set('GSP_SCTR', @@ -740,7 +773,7 @@ def _save_wavelength_calibrated(self, log.info('Wavelength-calibrated {:s} file saved to: ' '{:s} using reference lamp {:s}' ''.format(ccd.header['OBSTYPE'], - os.path.basename(new_filename), + os.path.basename(file_full_path), self.wcal_lamp_file)) ccd.header.set( 'GSP_LAMP', @@ -749,11 +782,11 @@ def _save_wavelength_calibrated(self, after='GSP_FLAT') write_fits(ccd=ccd, - full_path=new_filename, + full_path=file_full_path, parent_file=original_filename) - return os.path.basename(new_filename) + return file_full_path -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover sys.exit('This can not be run on its own.') diff --git a/goodman_pipeline/version.py b/goodman_pipeline/version.py index e36c285c..94cb01a6 100644 --- a/goodman_pipeline/version.py +++ b/goodman_pipeline/version.py @@ -1,2 +1,2 @@ # This is an automatic generated file please do not edit -__version__ = '1.3.0.dev21' \ No newline at end of file +__version__ = '1.3.0' \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index fb5c7d92..4598427b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,4 +32,4 @@ install_requires = ccdproc astroplan # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 1.3.0.dev21 +version = 1.3.0