From 9f95999865da9ccbc843998a109bf5b1b91f5a8d Mon Sep 17 00:00:00 2001 From: Pica Date: Thu, 19 Nov 2015 11:13:06 +0100 Subject: [PATCH 1/8] Store and Load function have been developed --- megaradrp/loader.py | 13 ++--- megaradrp/products.py | 20 ++++++- .../recipes/calibration/tests/test_arc.py | 7 ++- .../recipes/calibration/tests/test_bias.py | 4 +- megaradrp/recipes/tests/test_calibration.py | 29 +++++----- megaradrp/store.py | 53 ------------------- setup.py | 5 +- 7 files changed, 40 insertions(+), 91 deletions(-) delete mode 100644 megaradrp/store.py diff --git a/megaradrp/loader.py b/megaradrp/loader.py index 1ca38c32..087eeb30 100644 --- a/megaradrp/loader.py +++ b/megaradrp/loader.py @@ -1,5 +1,5 @@ # -# Copyright 2014 Universidad Complutense de Madrid +# Copyright 2014-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -17,17 +17,12 @@ # along with Megara DRP. If not, see . # -'''Load MEGARA DRP''' +"""Load MEGARA DRP""" from numina.core import drp_load -def megara_drp_load(): - '''Entry point to load MEGARA DRP.''' +def load_drp(): + """Entry point to load MEGARA DRP.""" return drp_load('megaradrp', 'drp.yaml') - -def load_cli_storage(): - '''Entry point to load storage functions for the CLI.''' - import megaradrp.store - return 0 diff --git a/megaradrp/products.py b/megaradrp/products.py index 3658cc33..9faf12ea 100644 --- a/megaradrp/products.py +++ b/megaradrp/products.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -17,7 +17,7 @@ # along with Megara DRP. If not, see . # -'''Products of the Megara Pipeline''' +"""Products of the Megara Pipeline""" ''' @@ -37,6 +37,7 @@ ''' +import yaml from numina.core import DataFrameType, DataProductType from numina.core.products import DataProductTag @@ -68,3 +69,18 @@ def __init__(self, default=None): super(TraceMap, self).__init__( ptype=dict, default=default) + def __numina_dump__(self, obj, where): + + filename = where.destination + '.yaml' + + with open(filename, 'w') as fd: + yaml.dump(obj, fd) + + return filename + + def __numina_load__(tag, obj): + + with open(obj, 'r') as fd: + traces = yaml.load(fd) + + return traces \ No newline at end of file diff --git a/megaradrp/recipes/calibration/tests/test_arc.py b/megaradrp/recipes/calibration/tests/test_arc.py index 6b7fdf92..e330fe20 100644 --- a/megaradrp/recipes/calibration/tests/test_arc.py +++ b/megaradrp/recipes/calibration/tests/test_arc.py @@ -23,8 +23,8 @@ from numina.user.cli import main -from megaradrp.loader import megara_drp_load -from megaradrp.loader import load_cli_storage +from megaradrp.loader import load_drp + BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/' @@ -38,7 +38,6 @@ def run_recipe(): @pytest.mark.usefixtures("numinatpldir") def test_mode_arc_calibration_set0(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) - load_cli_storage() + drpmocker.add_drp('MEGARA', load_drp) run_recipe() \ No newline at end of file diff --git a/megaradrp/recipes/calibration/tests/test_bias.py b/megaradrp/recipes/calibration/tests/test_bias.py index a0620a4f..283d02e4 100644 --- a/megaradrp/recipes/calibration/tests/test_bias.py +++ b/megaradrp/recipes/calibration/tests/test_bias.py @@ -9,7 +9,7 @@ from numina.core import ObservationResult from numina.core import DataFrame from megaradrp.recipes.calibration.bias import BiasRecipe -from megaradrp.loader import megara_drp_load +from megaradrp.loader import load_drp @pytest.mark.remote def test_bias(drpmocker): @@ -26,7 +26,7 @@ def test_bias(drpmocker): ob.mode = 'bias_image' ob.frames = [DataFrame(filename=f.name) for f in fs] - drpmocker.add_drp('MEGARA', megara_drp_load) + drpmocker.add_drp('MEGARA', load_drp) # Here we could directly import the required pipeline, # but the idea is to test all the process diff --git a/megaradrp/recipes/tests/test_calibration.py b/megaradrp/recipes/tests/test_calibration.py index 8197e02e..ee94c8b5 100644 --- a/megaradrp/recipes/tests/test_calibration.py +++ b/megaradrp/recipes/tests/test_calibration.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -17,19 +17,15 @@ # along with Megara DRP. If not, see . # -'''Tests for the calibration module.''' - -import os +"""Tests for the calibration module.""" import pytest -from numina.tests.testcache import download_cache - from numina.user.cli import main from numina.core import import_object from numina.core.pipeline import DrpSystem from megaradrp.recipes.calibration.bias import BiasRecipe -from megaradrp.loader import megara_drp_load, load_cli_storage +from megaradrp.loader import load_drp BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/' @@ -40,7 +36,7 @@ def run_recipe(): def test_recipe1(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) + drpmocker.add_drp('MEGARA', load_drp) insdrp = DrpSystem().query_by_name('MEGARA') pipeline = insdrp.pipelines.get('default') @@ -50,11 +46,12 @@ def test_recipe1(drpmocker): assert RecipeClass is BiasRecipe + @pytest.mark.remote @pytest.mark.usefixtures("numinatpldir") def test_mode_bias_set0(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) + drpmocker.add_drp('MEGARA', load_drp) run_recipe() @@ -62,25 +59,23 @@ def test_mode_bias_set0(drpmocker): @pytest.mark.remote @pytest.mark.usefixtures("numinatpldir") def test_mode_bias_set1(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) + + drpmocker.add_drp('MEGARA', load_drp) run_recipe() + @pytest.mark.remote @pytest.mark.usefixtures("numinatpldir") def test_mode_trace_map_set0(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) + drpmocker.add_drp('MEGARA', load_drp) run_recipe() + @pytest.mark.remote @pytest.mark.usefixtures("numinatpldir") def test_mode_fiber_flat_set0(drpmocker): - drpmocker.add_drp('MEGARA', megara_drp_load) - - # FIXME: this was completely crazy to debug - # if we don't load store/dump functions - # tracemap is loaded as a its filename - load_cli_storage() + drpmocker.add_drp('MEGARA', load_drp) run_recipe() \ No newline at end of file diff --git a/megaradrp/store.py b/megaradrp/store.py deleted file mode 100644 index 480eb6ba..00000000 --- a/megaradrp/store.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2011-2015 Universidad Complutense de Madrid -# -# This file is part of Megara DRP -# -# Megara DRP is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Megara DRP is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Megara DRP. If not, see . -# - -import logging - -import yaml - -from numina.store import dump, load - -from .products import TraceMap - -_logger = logging.getLogger('megaradrp.store') - - -_logger.debug('register dump functions') - - -@dump.register(TraceMap) -def _d(tag, obj, where): - - filename = where.destination + '.yaml' - - with open(filename, 'w') as fd: - yaml.dump(obj, fd) - - return filename - -_logger.debug('register load functions') - - -@load.register(TraceMap) -def _l(tag, obj): - - with open(obj, 'r') as fd: - traces = yaml.load(fd) - - return traces diff --git a/setup.py b/setup.py index 25936863..daa01943 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,8 @@ zip_safe=False, entry_points = { 'numina.pipeline.1': [ - 'MEGARA = megaradrp.loader:megara_drp_load', + 'MEGARA = megaradrp.loader:load_drp', ], - 'numina.storage.1': [ - 'megara_default = megaradrp.loader:load_cli_storage', - ] }, classifiers=[ "Programming Language :: Python :: 2.7", From c94fac1d2effed58aaed7e5317f934870bde30ec Mon Sep 17 00:00:00 2001 From: Pica Date: Wed, 14 Oct 2015 15:52:58 +0200 Subject: [PATCH 2/8] flatorg.py has been deleted --- megaradrp/recipes/calibration/flatorg.py | 154 ----------------------- 1 file changed, 154 deletions(-) delete mode 100644 megaradrp/recipes/calibration/flatorg.py diff --git a/megaradrp/recipes/calibration/flatorg.py b/megaradrp/recipes/calibration/flatorg.py deleted file mode 100644 index df2b1091..00000000 --- a/megaradrp/recipes/calibration/flatorg.py +++ /dev/null @@ -1,154 +0,0 @@ -# -# Copyright 2011-2014 Universidad Complutense de Madrid -# -# This file is part of Megara DRP -# -# Megara DRP is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Megara DRP is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Megara DRP. If not, see . -# - -'''Original fiber flat calibration recipes''' - -import logging - -import numpy -from astropy.io import fits - -from numina.core import Product -from numina.core.products import ArrayType -from numina.core.requirements import ObservationResultRequirement -from numina.array.combine import median as c_median -from numina.flow import SerialFlow -from numina.flow.processing import BiasCorrector - -from megaradrp.core import MegaraBaseRecipe -from megaradrp.processing import OverscanCorrector, TrimImage -from megaradrp.core import peakdet -from megaradrp.products import MasterFiberFlat -from megaradrp.requirements import MasterBiasRequirement - -_logger = logging.getLogger('numina.recipes.megara') - - -class FiberFlatRecipe(MegaraBaseRecipe): - - '''Process FIBER_FLAT images and create MASTER_FIBER_FLAT.''' - - master_bias = MasterBiasRequirement() - obresult = ObservationResultRequirement() - - fiberflat_frame = Product(MasterFiberFlat) - fiberflat_rss = Product(MasterFiberFlat) - traces = Product(ArrayType) - - def __init__(self): - super(FiberFlatRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - _logger.info('starting fiber flat reduction') - - o_c = OverscanCorrector() - t_i = TrimImage() - - with rinput.master_bias.open() as hdul: - mbias = hdul[0].data.copy() - b_c = BiasCorrector(mbias) - - basicflow = SerialFlow([o_c, t_i, b_c]) - - cdata = [] - - try: - for frame in rinput.obresult.images: - hdulist = frame.open() - hdulist = basicflow(hdulist) - cdata.append(hdulist) - - _logger.info('stacking %d images using median', len(cdata)) - - data = c_median([d[0].data for d in cdata], dtype='float32') - template_header = cdata[0][0].header - hdu = fits.PrimaryHDU(data[0], header=template_header) - finally: - for hdulist in cdata: - hdulist.close() - - hdr = hdu.header - hdr['IMGTYP'] = ('FIBER_FLAT', 'Image type') - hdr['NUMTYP'] = ('MASTER_FIBER_FLAT', 'Data product type') - hdr = self.set_base_headers(hdr) - hdr['CCDMEAN'] = data[0].mean() - - varhdu = fits.ImageHDU(data[1], name='VARIANCE') - num = fits.ImageHDU(data[2], name='MAP') - hdulist = fits.HDUList([hdu, varhdu, num]) - - # Trace extract and normalize - # Cut a region in the center - mm = data[0] - - cut = mm[:, 1980:2020] - colcut = cut.sum(axis=1) / 40.0 - - # Find peaks - maxt, mint = peakdet(v=colcut, delta=0.3, back=5e3) - _logger.info('found %d peaks', len(maxt)) - # Cut around the peak - - # Maximum half width of peaks - maxw = 3 - - borders = numpy.empty((maxt.shape[0], 3), dtype='int') - borders[:, 1] = maxt[:, 0] - borders[1:, 0] = mint[:-1, 0] - borders[0, 0] = 0 - borders[:-1, 2] = mint[1:, 0] - borders[-1, 2] = 1e4 - - borders[:, 2] = numpy.minimum(borders[:, 2], borders[:, 1] + maxw) - borders[:, 0] = numpy.maximum(borders[:, 0], borders[:, 1] - maxw) - - _logger.info('extract fibers') - rss = numpy.empty((borders.shape[0], mm.shape[1])) - for idx, r in enumerate(borders): - l = int(r[0]) - r = int(r[2]) + 1 - sl = (slice(l, r), ) - m = mm[sl].sum(axis=0) - rss[idx] = m - - # Normalize RSS - _logger.info('normalize fibers') - nidx = 350 - single_s = rss[nidx, :] - - # FIXME: hardcoded values - x_fit_min = 50 - x_fit_max = 4050 # This is the last value, not included - x_fit = range(x_fit_min, x_fit_max) - - # fitting a 3rd degree polynomial - pol_coeff = numpy.polyfit(x_fit, single_s[x_fit_min:x_fit_max], deg=3) - pol_fit = numpy.poly1d(pol_coeff) - norm = pol_fit(range(single_s.shape[0])) - rss_norm = rss / norm - - _logger.info('fiber flat reduction ended') - - result = self.create_result(fiberflat_frame=hdu, - fiberflat_rss=fits.PrimaryHDU(rss_norm), - traces=borders) - - return result From 155acdef9877da386dbe0225910d97f96ca8e4f2 Mon Sep 17 00:00:00 2001 From: Pica Date: Thu, 19 Nov 2015 11:57:07 +0100 Subject: [PATCH 3/8] Remove DataProductRequirement --- megaradrp/recipes/calibration/base.py | 8 ++++---- megaradrp/recipes/scientific.py | 8 ++++---- megaradrp/requirements.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/megaradrp/recipes/calibration/base.py b/megaradrp/recipes/calibration/base.py index d0c02df8..19dd4974 100644 --- a/megaradrp/recipes/calibration/base.py +++ b/megaradrp/recipes/calibration/base.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -17,7 +17,7 @@ # along with Megara DRP. If not, see . # -'''Calibration Recipes for Megara''' +"""Calibration Recipes for Megara""" import logging @@ -29,7 +29,7 @@ from astropy import wcs -from numina.core import Product, DataProductRequirement, Requirement +from numina.core import Product, Requirement from numina.core.products import ArrayType from numina.core.requirements import ObservationResultRequirement from numina.array.combine import median as c_median @@ -81,7 +81,7 @@ class PseudoFluxCalibrationRecipe(MegaraBaseRecipe): master_bias = MasterBiasRequirement() master_fiber_flat = MasterFiberFlatRequirement() traces = Requirement(TraceMap, 'Trace information of the Apertures') - reference_spectrum = DataProductRequirement( + reference_spectrum = Requirement( MasterFiberFlat, 'Reference spectrum') calibration = Product(MasterSensitivity) diff --git a/megaradrp/recipes/scientific.py b/megaradrp/recipes/scientific.py index d31f41e9..f5e35212 100644 --- a/megaradrp/recipes/scientific.py +++ b/megaradrp/recipes/scientific.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -23,7 +23,7 @@ from astropy.io import fits -from numina.core import Product, DataProductRequirement +from numina.core import Product, Requirement from numina.core.requirements import ObservationResultRequirement, Requirement from numina.array.combine import median as c_median from numina.flow import SerialFlow @@ -52,7 +52,7 @@ class FiberMOSRecipe(MegaraBaseRecipe): master_bias = MasterBiasRequirement() master_fiber_flat = MasterFiberFlatRequirement() traces = Requirement(TraceMap, 'Trace information of the Apertures') - sensitivity = DataProductRequirement( + sensitivity = Requirement( MasterSensitivity, 'Sensitivity', optional=True) # Products @@ -160,7 +160,7 @@ class FiberMOSRecipe2(MegaraBaseRecipe): master_bias = MasterBiasRequirement() master_fiber_flat = MasterFiberFlatRequirement() traces = Requirement(TraceMap, 'Trace information of the Apertures') - sensitivity = DataProductRequirement( + sensitivity = Requirement( MasterSensitivity, 'Sensitivity', optional=True) # Products diff --git a/megaradrp/requirements.py b/megaradrp/requirements.py index 59c9195c..c228053d 100644 --- a/megaradrp/requirements.py +++ b/megaradrp/requirements.py @@ -17,14 +17,14 @@ # along with Megara DRP. If not, see . # -'''Typical requirements of recipes''' +"""Typical requirements of recipes""" -from numina.core import DataProductRequirement +from numina.core import Requirement from .products import MasterBias, MasterDark, MasterFiberFlat -class MasterBiasRequirement(DataProductRequirement): +class MasterBiasRequirement(Requirement): def __init__(self, optional=False): super(MasterBiasRequirement, self).__init__(MasterBias, @@ -33,13 +33,13 @@ def __init__(self, optional=False): ) -class MasterDarkRequirement(DataProductRequirement): +class MasterDarkRequirement(Requirement): def __init__(self): super(MasterDarkRequirement, self).__init__(MasterDark, 'Master DARK image') -class MasterFiberFlatRequirement(DataProductRequirement): +class MasterFiberFlatRequirement(Requirement): def __init__(self): super(MasterFiberFlatRequirement, self).__init__(MasterFiberFlat, From e4d003b40bca0320522df6caece1632ffc5a3976 Mon Sep 17 00:00:00 2001 From: Pica Date: Thu, 19 Nov 2015 15:34:51 +0100 Subject: [PATCH 4/8] Add tests for taggers --- megaradrp/tests/__init__.py | 0 megaradrp/tests/test_taggers.py | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 megaradrp/tests/__init__.py create mode 100644 megaradrp/tests/test_taggers.py diff --git a/megaradrp/tests/__init__.py b/megaradrp/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megaradrp/tests/test_taggers.py b/megaradrp/tests/test_taggers.py new file mode 100644 index 00000000..6b68ec5e --- /dev/null +++ b/megaradrp/tests/test_taggers.py @@ -0,0 +1,81 @@ +# +# Copyright 2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +import numina.core + +import astropy.io.fits as fits + +from megaradrp.taggers import tagger_empty, tagger_vph + +def test_tagger_empty(): + + img1 = fits.PrimaryHDU(data=[1,2,3], header=fits.Header()) + img1.header['FILTER'] = 'FILTER-A' + img1.header['READM'] = 'MOD1' + frame1 = numina.core.DataFrame(frame=fits.HDUList(img1)) + + ob = numina.core.ObservationResult() + ob.frames = [frame1] + + tags = tagger_empty(ob) + + assert tags == {} + + +def test_tagger_empty_no_images(): + + ob = numina.core.ObservationResult() + + tags = tagger_empty(ob) + + assert tags == {} + + +def test_tagger_vph(): + + img1 = fits.PrimaryHDU(data=[1,2,3], header=fits.Header()) + img1.header['VPH'] = 'VPH405_LR' + img1.header['READM'] = 'MOD1' + frame1 = numina.core.DataFrame(frame=fits.HDUList(img1)) + + img2 = fits.PrimaryHDU(data=[1,2,3], header=fits.Header()) + img2.header['VPH'] = 'VPH405_LR' + img2.header['READM'] = 'MOD1' + frame2 = numina.core.DataFrame(frame=fits.HDUList(img2)) + + ob = numina.core.ObservationResult() + ob.frames = [frame1, frame2] + + tags = tagger_vph(ob) + + assert tags == {'vph': 'VPH405_LR'} + + +def test_tagger_vph_no_images(): + + ob = numina.core.ObservationResult() + + tags = tagger_vph(ob) + + assert tags == {} + + +if __name__ == "__main__": + test_tagger_vph() + From c480c786e260860f6c785458a1b90d6656f0f023 Mon Sep 17 00:00:00 2001 From: Sergio Pascual Date: Thu, 19 Nov 2015 16:08:16 +0100 Subject: [PATCH 5/8] Organize recipes, do not load recipes in __init__.py --- megaradrp/drp.yaml | 14 +- megaradrp/recipes/__init__.py | 21 --- megaradrp/recipes/calibration/__init__.py | 28 ---- megaradrp/recipes/calibration/base.py | 185 +-------------------- megaradrp/recipes/calibration/bpm.py | 48 ++++++ megaradrp/recipes/calibration/dark.py | 53 ++++++ megaradrp/recipes/calibration/flat.py | 95 +---------- megaradrp/recipes/calibration/fluxcal.py | 151 +++++++++++++++++ megaradrp/recipes/calibration/linearity.py | 46 +++++ megaradrp/recipes/calibration/trace.py | 98 +++++++++++ megaradrp/recipes/calibration/twilight.py | 48 ++++++ megaradrp/recipes/experimental.py | 20 --- megaradrp/recipes/scientific.py | 2 - 13 files changed, 451 insertions(+), 358 deletions(-) create mode 100644 megaradrp/recipes/calibration/bpm.py create mode 100644 megaradrp/recipes/calibration/dark.py create mode 100644 megaradrp/recipes/calibration/fluxcal.py create mode 100644 megaradrp/recipes/calibration/linearity.py create mode 100644 megaradrp/recipes/calibration/trace.py create mode 100644 megaradrp/recipes/calibration/twilight.py delete mode 100644 megaradrp/recipes/experimental.py diff --git a/megaradrp/drp.yaml b/megaradrp/drp.yaml index 888e713d..90f9a738 100644 --- a/megaradrp/drp.yaml +++ b/megaradrp/drp.yaml @@ -82,21 +82,17 @@ modes: tagger: megaradrp.taggers.tagger_vph pipelines: default: + version: 1 recipes: bias_image: megaradrp.recipes.calibration.bias.BiasRecipe - dark_image: megaradrp.recipes.DarkRecipe - fiber_flat_image: megaradrp.recipes.FiberFlatRecipe + dark_image: megaradrp.recipes.calibration.dark.DarkRecipe + fiber_flat_image: megaradrp.recipes.calibration.flat.FiberFlatRecipe mos_image: megaradrp.recipes.scientific.FiberMOSRecipe2 - flux_calibration: megaradrp.recipes.calibration.PseudoFluxCalibrationRecipe - trace_map: megaradrp.recipes.calibration.flat.TraceMapRecipe + flux_calibration: megaradrp.recipes.calibration.fluxcal.PseudoFluxCalibrationRecipe + trace_map: megaradrp.recipes.calibration.trace.TraceMapRecipe arc_calibration: megaradrp.recipes.calibration.arc.ArcCalibrationRecipe fail: numina.core.utils.AlwaysFailRecipe success: numina.core.utils.AlwaysSuccessRecipe - version: 1 - experimental: - recipes: - fiber_flat_image: megaradrp.recipes.calibration.flat.FiberFlatRecipe - version: 1 products: - name: megaradrp.products.TraceMap alias: TraceMap diff --git a/megaradrp/recipes/__init__.py b/megaradrp/recipes/__init__.py index 082b2155..e69de29b 100644 --- a/megaradrp/recipes/__init__.py +++ b/megaradrp/recipes/__init__.py @@ -1,21 +0,0 @@ -# -# Copyright 2011-2014 Universidad Complutense de Madrid -# -# This file is part of Megara DRP -# -# Megara DRP is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Megara DRP is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Megara DRP. If not, see . -# - -from .calibration import DarkRecipe, FiberFlatRecipe -from .scientific import FiberMOSRecipe \ No newline at end of file diff --git a/megaradrp/recipes/calibration/__init__.py b/megaradrp/recipes/calibration/__init__.py index 6140ad49..e69de29b 100644 --- a/megaradrp/recipes/calibration/__init__.py +++ b/megaradrp/recipes/calibration/__init__.py @@ -1,28 +0,0 @@ -# -# Copyright 2011-2014 Universidad Complutense de Madrid -# -# This file is part of Megara DRP -# -# Megara DRP is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Megara DRP is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Megara DRP. If not, see . -# - -'''Calibration Recipes for Megara''' - -from .base import DarkRecipe -from .base import PseudoFluxCalibrationRecipe -from .flat import TraceMapRecipe, TwilightFiberFlatRecipe -from .flat import FiberFlatRecipe -from .base import LCB_IFU_StdStarRecipe, FiberMOS_StdStarRecipe -from .base import SensitivityFromStdStarRecipe, S_And_E_FromStdStarsRecipe -from .base import BadPixelsMaskRecipe, LinearityTestRecipe \ No newline at end of file diff --git a/megaradrp/recipes/calibration/base.py b/megaradrp/recipes/calibration/base.py index 19dd4974..4f4f7111 100644 --- a/megaradrp/recipes/calibration/base.py +++ b/megaradrp/recipes/calibration/base.py @@ -21,167 +21,18 @@ import logging -import numpy - -from scipy.interpolate import interp1d - -from astropy.io import fits -from astropy import wcs - - from numina.core import Product, Requirement from numina.core.products import ArrayType from numina.core.requirements import ObservationResultRequirement -from numina.array.combine import median as c_median -from numina.flow import SerialFlow -from numina.flow.processing import BiasCorrector from megaradrp.core import MegaraBaseRecipe -from megaradrp.processing import OverscanCorrector, TrimImage -from megaradrp.processing import ApertureExtractor, FiberFlatCorrector -# from numina.logger import log_to_history from megaradrp.requirements import MasterBiasRequirement -from megaradrp.requirements import MasterFiberFlatRequirement -from megaradrp.products import MasterBias, MasterDark, MasterFiberFlat -from megaradrp.products import TraceMap, MasterSensitivity +from megaradrp.products import MasterFiberFlat _logger = logging.getLogger('numina.recipes.megara') - -class DarkRecipe(MegaraBaseRecipe): - - '''Process DARK images and provide MASTER_DARK. ''' - - master_bias = MasterBiasRequirement() - - darkframe = Product(MasterDark) - - def __init__(self): - super(DarkRecipe, self).__init__( - version="0.1.0" - ) - - # FIXME find a better way of doing this automatically - # @log_to_history(_logger) - def run(self, rinput): - - _logger.info('starting dark reduction') - - _logger.info('dark reduction ended') - - result = self.create_result(darkframe=None) - return result - - -class PseudoFluxCalibrationRecipe(MegaraBaseRecipe): - - obresult = ObservationResultRequirement() - master_bias = MasterBiasRequirement() - master_fiber_flat = MasterFiberFlatRequirement() - traces = Requirement(TraceMap, 'Trace information of the Apertures') - reference_spectrum = Requirement( - MasterFiberFlat, 'Reference spectrum') - - calibration = Product(MasterSensitivity) - calibration_rss = Product(MasterSensitivity) - - def __init__(self): - super(PseudoFluxCalibrationRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - _logger.info('starting pseudo flux calibration') - - o_c = OverscanCorrector() - t_i = TrimImage() - - with rinput.master_bias.open() as hdul: - mbias = hdul[0].data.copy() - b_c = BiasCorrector(mbias) - - a_e = ApertureExtractor(rinput.traces) - - with rinput.master_fiber_flat.open() as hdul: - f_f_c = FiberFlatCorrector(hdul) - - basicflow = SerialFlow([o_c, t_i, b_c, a_e, f_f_c]) - - t_data = [] - - try: - for frame in rinput.obresult.images: - hdulist = frame.open() - hdulist = basicflow(hdulist) - t_data.append(hdulist) - - data_t = c_median([d[0].data for d in t_data], dtype='float32') - template_header = t_data[0][0].header - hdu_t = fits.PrimaryHDU(data_t[0], header=template_header) - finally: - for hdulist in t_data: - hdulist.close() - - hdr = hdu_t.header - hdr = self.set_base_headers(hdr) - hdr['CCDMEAN'] = data_t[0].mean() - hdr['NUMTYP'] = ('SCIENCE_TARGET', 'Data product type') - - # FIXME: hardcoded calibration - # Polynomial that translates pixels to wl - _logger.warning('using hardcoded LR-U spectral calibration') - wlcal = [7.12175997e-10, -9.36387541e-06, - 2.13624855e-01, 3.64665269e+03] - plin = numpy.poly1d(wlcal) - wl_n_r = plin(range(1, hdu_t.data.shape[1] + 1)) # Non-regular WL - - _logger.info('resampling reference spectrum') - - wlr = [3673.12731884058, 4417.497427536232] - size = hdu_t.data.shape[1] - delt = (wlr[1] - wlr[0]) / (size - 1) - - def add_wcs(hdr): - hdr['CRPIX1'] = 1 - hdr['CRVAL1'] = wlr[0] - hdr['CDELT1'] = delt - hdr['CTYPE1'] = 'WAVELENGTH' - hdr['CRPIX2'] = 1 - hdr['CRVAL2'] = 1 - hdr['CDELT2'] = 1 - hdr['CTYPE2'] = 'PIXEL' - return hdr - - with rinput.reference_spectrum.open() as hdul: - # Needs resampling - data = hdul[0].data - w_ref = wcs.WCS(hdul[0].header) - # FIXME: Hardcoded values - # because we do not have WL calibration - pix = range(1, len(data) + 1) - wl = w_ref.wcs_pix2world(pix, 1) - # The 0 mean 0-based - si = interp1d(wl, data) - # Reference spectrum evaluated in the irregular WL grid - final = si(wl_n_r) - - sens_data = final / hdu_t.data - hdu_sens = fits.PrimaryHDU(sens_data, header=hdu_t.header) - - # Very simple wl calibration - # add_wcs(hdu_sens.header) - - # add_wcs(hdu_t.header) - - _logger.info('pseudo flux calibration reduction ended') - - result = self.create_result( - calibration=hdu_sens, calibration_rss=hdu_t) - return result - - class LCB_IFU_StdStarRecipe(MegaraBaseRecipe): master_bias = MasterBiasRequirement() @@ -252,37 +103,3 @@ def __init__(self): def run(self, rinput): pass - - -class BadPixelsMaskRecipe(MegaraBaseRecipe): - - '''Process BIAS images and create MASTER_BIAS.''' - - obresult = ObservationResultRequirement() - - biasframe = Product(MasterBias) - - def __init__(self): - super(BadPixelsMaskRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - pass - - -class LinearityTestRecipe(MegaraBaseRecipe): - - '''Process BIAS images and create MASTER_BIAS.''' - - obresult = ObservationResultRequirement() - - biasframe = Product(MasterBias) - - def __init__(self): - super(LinearityTestRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - pass diff --git a/megaradrp/recipes/calibration/bpm.py b/megaradrp/recipes/calibration/bpm.py new file mode 100644 index 00000000..2bfd9f28 --- /dev/null +++ b/megaradrp/recipes/calibration/bpm.py @@ -0,0 +1,48 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +"""Bad PIxel Mask (BPM) recipe""" + +import logging + +from numina.core import Product +from numina.core.requirements import ObservationResultRequirement + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.products import MasterBias + + +_logger = logging.getLogger('numina.recipes.megara') + + +class BadPixelsMaskRecipe(MegaraBaseRecipe): + + '''Process BIAS images and create MASTER_BIAS.''' + + obresult = ObservationResultRequirement() + + biasframe = Product(MasterBias) + + def __init__(self): + super(BadPixelsMaskRecipe, self).__init__( + version="0.1.0" + ) + + def run(self, rinput): + pass diff --git a/megaradrp/recipes/calibration/dark.py b/megaradrp/recipes/calibration/dark.py new file mode 100644 index 00000000..4cb4b49a --- /dev/null +++ b/megaradrp/recipes/calibration/dark.py @@ -0,0 +1,53 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +"""Dark current calibration Recipe for Megara""" + +import logging + +from numina.core import Product + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.requirements import MasterBiasRequirement +from megaradrp.products import MasterDark + + +class DarkRecipe(MegaraBaseRecipe): + + '''Process DARK images and provide MASTER_DARK. ''' + + master_bias = MasterBiasRequirement() + + darkframe = Product(MasterDark) + + def __init__(self): + super(DarkRecipe, self).__init__( + version="0.1.0" + ) + + + def run(self, rinput): + logger = logging.getLogger('numina.megara.recipes' + ) + logger.info('starting dark reduction') + + logger.info('dark reduction ended') + + result = self.create_result(darkframe=None) + return result diff --git a/megaradrp/recipes/calibration/flat.py b/megaradrp/recipes/calibration/flat.py index 4df61d07..45189eeb 100644 --- a/megaradrp/recipes/calibration/flat.py +++ b/megaradrp/recipes/calibration/flat.py @@ -17,17 +17,15 @@ # along with Megara DRP. If not, see . # -'''Calibration Recipes for Megara''' +"""Calibration Recipes for Megara""" from __future__ import division, print_function import logging -import numpy from astropy.io import fits from numina.core import Product, Requirement -from numina.core.products import ArrayType from numina.core.requirements import ObservationResultRequirement from numina.array.combine import median as c_median from numina.flow import SerialFlow @@ -35,15 +33,9 @@ from megaradrp.core import MegaraBaseRecipe from megaradrp.processing import OverscanCorrector, TrimImage -# from numina.logger import log_to_history - from megaradrp.products import MasterFiberFlat from megaradrp.products import TraceMap from megaradrp.requirements import MasterBiasRequirement - -from megaradrp.trace.traces import init_traces -#from megaradrp.trace._traces import tracing # @UnresolvedImport -from numina.array.trace.traces import trace from megaradrp.core import apextract_tracemap _logger = logging.getLogger('numina.recipes.megara') @@ -129,88 +121,3 @@ def run(self, rinput): fiberflat_rss=rss) return result - - -class TwilightFiberFlatRecipe(MegaraBaseRecipe): - - master_bias = MasterBiasRequirement() - obresult = ObservationResultRequirement() - - fiberflat_frame = Product(MasterFiberFlat) - fiberflat_rss = Product(MasterFiberFlat) - traces = Product(ArrayType) - - def __init__(self): - super(TwilightFiberFlatRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - pass - - -class TraceMapRecipe(MegaraBaseRecipe): - - obresult = ObservationResultRequirement() - master_bias = MasterBiasRequirement() - fiberflat_frame = Product(MasterFiberFlat) - traces = Product(TraceMap) - - def __init__(self): - super(TraceMapRecipe, self).__init__( - version="0.1.0" - ) - - def run(self, rinput): - - result = self.process_base(rinput.obresult, rinput.master_bias) - - data = result[0].data - - # fit_traces = domefun(data, cstart=2000, hs=20) - - cstart = 2000 - hs = 3 - step1 = 2 - background1 = 150.0 - maxdis1 = 2.0 - - _logger.info('find peaks in column %i', cstart) - - central_peaks = init_traces(data, center=cstart, hs=hs, - background=background1) - - _logger.info(' %i peaks found', len(central_peaks)) - - tracelist = [] - if data.dtype.byteorder != '=': - _logger.debug('byteswapping image') - image2 = data.byteswap().newbyteorder() - else: - image2 = data - - _logger.info('trace peaks') - for dtrace in central_peaks.values(): - - mm = trace(image2, x=cstart, y=dtrace.start[1], step=step1, - hs=hs, background=background1, maxdis=maxdis1) - - pfit = numpy.polyfit(mm[:,0], mm[:,1], deg=5) - - tracelist.append({'fibid': dtrace.fibid, 'boxid': dtrace.boxid, - 'start':0, 'stop':4095, - 'fitparms': pfit.tolist()}) - - #plt.title('cython version, fiber %i' % trace.fibid) - #plt.plot(mm[:,0], mm[:,1], 'r*') - #xpix = np.arange(0, 2000, 1) - #p = numpy.poly1d(pfit) - #plt.plot(xpix, p(xpix), 'b') - #plt.show() - - return self.create_result(fiberflat_frame=result, - traces=tracelist) - - def process_base(self, obresult, master_bias): - reduced = process_common(self, obresult, master_bias) - return reduced diff --git a/megaradrp/recipes/calibration/fluxcal.py b/megaradrp/recipes/calibration/fluxcal.py new file mode 100644 index 00000000..08cae235 --- /dev/null +++ b/megaradrp/recipes/calibration/fluxcal.py @@ -0,0 +1,151 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +"""Calibration Recipes for Megara""" + +import logging + +import numpy + +from scipy.interpolate import interp1d + +from astropy.io import fits +from astropy import wcs + +from numina.core import Product, Requirement +from numina.core.requirements import ObservationResultRequirement +from numina.array.combine import median as c_median +from numina.flow import SerialFlow +from numina.flow.processing import BiasCorrector + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.processing import OverscanCorrector, TrimImage +from megaradrp.processing import ApertureExtractor, FiberFlatCorrector +from megaradrp.requirements import MasterBiasRequirement +from megaradrp.requirements import MasterFiberFlatRequirement +from megaradrp.products import MasterFiberFlat +from megaradrp.products import TraceMap, MasterSensitivity + +_logger = logging.getLogger('numina.recipes.megara') + +class PseudoFluxCalibrationRecipe(MegaraBaseRecipe): + + obresult = ObservationResultRequirement() + master_bias = MasterBiasRequirement() + master_fiber_flat = MasterFiberFlatRequirement() + traces = Requirement(TraceMap, 'Trace information of the Apertures') + reference_spectrum = Requirement( + MasterFiberFlat, 'Reference spectrum') + + calibration = Product(MasterSensitivity) + calibration_rss = Product(MasterSensitivity) + + def __init__(self): + super(PseudoFluxCalibrationRecipe, self).__init__( + version="0.1.0" + ) + + def run(self, rinput): + _logger.info('starting pseudo flux calibration') + + o_c = OverscanCorrector() + t_i = TrimImage() + + with rinput.master_bias.open() as hdul: + mbias = hdul[0].data.copy() + b_c = BiasCorrector(mbias) + + a_e = ApertureExtractor(rinput.traces) + + with rinput.master_fiber_flat.open() as hdul: + f_f_c = FiberFlatCorrector(hdul) + + basicflow = SerialFlow([o_c, t_i, b_c, a_e, f_f_c]) + + t_data = [] + + try: + for frame in rinput.obresult.images: + hdulist = frame.open() + hdulist = basicflow(hdulist) + t_data.append(hdulist) + + data_t = c_median([d[0].data for d in t_data], dtype='float32') + template_header = t_data[0][0].header + hdu_t = fits.PrimaryHDU(data_t[0], header=template_header) + finally: + for hdulist in t_data: + hdulist.close() + + hdr = hdu_t.header + hdr = self.set_base_headers(hdr) + hdr['CCDMEAN'] = data_t[0].mean() + hdr['NUMTYP'] = ('SCIENCE_TARGET', 'Data product type') + + # FIXME: hardcoded calibration + # Polynomial that translates pixels to wl + _logger.warning('using hardcoded LR-U spectral calibration') + wlcal = [7.12175997e-10, -9.36387541e-06, + 2.13624855e-01, 3.64665269e+03] + plin = numpy.poly1d(wlcal) + wl_n_r = plin(range(1, hdu_t.data.shape[1] + 1)) # Non-regular WL + + _logger.info('resampling reference spectrum') + + wlr = [3673.12731884058, 4417.497427536232] + size = hdu_t.data.shape[1] + delt = (wlr[1] - wlr[0]) / (size - 1) + + def add_wcs(hdr): + hdr['CRPIX1'] = 1 + hdr['CRVAL1'] = wlr[0] + hdr['CDELT1'] = delt + hdr['CTYPE1'] = 'WAVELENGTH' + hdr['CRPIX2'] = 1 + hdr['CRVAL2'] = 1 + hdr['CDELT2'] = 1 + hdr['CTYPE2'] = 'PIXEL' + return hdr + + with rinput.reference_spectrum.open() as hdul: + # Needs resampling + data = hdul[0].data + w_ref = wcs.WCS(hdul[0].header) + # FIXME: Hardcoded values + # because we do not have WL calibration + pix = range(1, len(data) + 1) + wl = w_ref.wcs_pix2world(pix, 1) + # The 0 mean 0-based + si = interp1d(wl, data) + # Reference spectrum evaluated in the irregular WL grid + final = si(wl_n_r) + + sens_data = final / hdu_t.data + hdu_sens = fits.PrimaryHDU(sens_data, header=hdu_t.header) + + # Very simple wl calibration + # add_wcs(hdu_sens.header) + + # add_wcs(hdu_t.header) + + _logger.info('pseudo flux calibration reduction ended') + + result = self.create_result( + calibration=hdu_sens, calibration_rss=hdu_t) + return result diff --git a/megaradrp/recipes/calibration/linearity.py b/megaradrp/recipes/calibration/linearity.py new file mode 100644 index 00000000..271e570e --- /dev/null +++ b/megaradrp/recipes/calibration/linearity.py @@ -0,0 +1,46 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +"""Calibration Recipes for Megara""" + +import logging + +from numina.core import Product +from numina.core.requirements import ObservationResultRequirement + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.products import MasterBias + +_logger = logging.getLogger('numina.recipes.megara') + +class LinearityTestRecipe(MegaraBaseRecipe): + + """Process LINTEST images and create LINEARITY_CORRECTON.""" + + obresult = ObservationResultRequirement() + + biasframe = Product(MasterBias) + + def __init__(self): + super(LinearityTestRecipe, self).__init__( + version="0.1.0" + ) + + def run(self, rinput): + pass diff --git a/megaradrp/recipes/calibration/trace.py b/megaradrp/recipes/calibration/trace.py new file mode 100644 index 00000000..1b8eadf0 --- /dev/null +++ b/megaradrp/recipes/calibration/trace.py @@ -0,0 +1,98 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +"""Fiber tracing Recipe.""" + +from __future__ import division, print_function + +import logging +import numpy + +from numina.array.trace.traces import trace +from numina.core import Product +from numina.core.requirements import ObservationResultRequirement + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.products import MasterFiberFlat +from megaradrp.products import TraceMap +from megaradrp.requirements import MasterBiasRequirement +from megaradrp.trace.traces import init_traces +from .flat import process_common + +_logger = logging.getLogger('numina.recipes.megara') + + +class TraceMapRecipe(MegaraBaseRecipe): + + obresult = ObservationResultRequirement() + master_bias = MasterBiasRequirement() + fiberflat_frame = Product(MasterFiberFlat) + traces = Product(TraceMap) + + def __init__(self): + super(TraceMapRecipe, self).__init__( + version="0.1.0" + ) + + def run(self, rinput): + + result = self.process_base(rinput.obresult, rinput.master_bias) + + data = result[0].data + + # fit_traces = domefun(data, cstart=2000, hs=20) + + cstart = 2000 + hs = 3 + step1 = 2 + background1 = 150.0 + maxdis1 = 2.0 + + _logger.info('find peaks in column %i', cstart) + + central_peaks = init_traces(data, center=cstart, hs=hs, + background=background1) + + _logger.info(' %i peaks found', len(central_peaks)) + + tracelist = [] + if data.dtype.byteorder != '=': + _logger.debug('byteswapping image') + image2 = data.byteswap().newbyteorder() + else: + image2 = data + + _logger.info('trace peaks') + for dtrace in central_peaks.values(): + + mm = trace(image2, x=cstart, y=dtrace.start[1], step=step1, + hs=hs, background=background1, maxdis=maxdis1) + + pfit = numpy.polyfit(mm[:,0], mm[:,1], deg=5) + + tracelist.append({'fibid': dtrace.fibid, 'boxid': dtrace.boxid, + 'start':0, 'stop':4095, + 'fitparms': pfit.tolist()}) + + return self.create_result(fiberflat_frame=result, + traces=tracelist) + + def process_base(self, obresult, master_bias): + reduced = process_common(self, obresult, master_bias) + return reduced diff --git a/megaradrp/recipes/calibration/twilight.py b/megaradrp/recipes/calibration/twilight.py new file mode 100644 index 00000000..ee5ed8f5 --- /dev/null +++ b/megaradrp/recipes/calibration/twilight.py @@ -0,0 +1,48 @@ +# +# Copyright 2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +""" Twilight fiber flat Calibration Recipes for Megara""" + +from __future__ import division, print_function + +from numina.core import Product +from numina.core.products import ArrayType +from numina.core.requirements import ObservationResultRequirement + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.products import MasterFiberFlat +from megaradrp.requirements import MasterBiasRequirement + + +class TwilightFiberFlatRecipe(MegaraBaseRecipe): + + master_bias = MasterBiasRequirement() + obresult = ObservationResultRequirement() + + fiberflat_frame = Product(MasterFiberFlat) + fiberflat_rss = Product(MasterFiberFlat) + traces = Product(ArrayType) + + def __init__(self): + super(TwilightFiberFlatRecipe, self).__init__( + version="0.1.0" + ) + + def run(self, rinput): + pass diff --git a/megaradrp/recipes/experimental.py b/megaradrp/recipes/experimental.py deleted file mode 100644 index d5bbfd7f..00000000 --- a/megaradrp/recipes/experimental.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2011-2014 Universidad Complutense de Madrid -# -# This file is part of Megara DRP -# -# Megara DRP is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Megara DRP is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Megara DRP. If not, see . -# - -'''Recipes with experimental features.''' diff --git a/megaradrp/recipes/scientific.py b/megaradrp/recipes/scientific.py index f5e35212..b0483010 100644 --- a/megaradrp/recipes/scientific.py +++ b/megaradrp/recipes/scientific.py @@ -33,8 +33,6 @@ from megaradrp.processing import OverscanCorrector, TrimImage from megaradrp.processing import ApertureExtractor, FiberFlatCorrector from megaradrp.processing import ApertureExtractor2 - -# from numina.logger import log_to_history from megaradrp.requirements import MasterBiasRequirement from megaradrp.requirements import MasterFiberFlatRequirement from megaradrp.products import MasterFiberFlat From a0d22c1dbe6a486d4dd72337fac2e607859f6bb1 Mon Sep 17 00:00:00 2001 From: Sergio Pascual Date: Thu, 19 Nov 2015 16:50:11 +0100 Subject: [PATCH 6/8] Split contents of core.py: * "create" goes to test, is a function to create images * MegaraBaseRecipe goes to core.recipe * The rest go to core.processing --- megaradrp/core/__init__.py | 3 + megaradrp/{core.py => core/processing.py} | 91 ++----------------- megaradrp/core/recipe.py | 31 +++++++ megaradrp/processing/aperture.py | 9 +- megaradrp/processing/trimover.py | 2 +- megaradrp/tests/create.py | 101 ++++++++++++++++++++++ 6 files changed, 147 insertions(+), 90 deletions(-) create mode 100644 megaradrp/core/__init__.py rename megaradrp/{core.py => core/processing.py} (65%) create mode 100644 megaradrp/core/recipe.py create mode 100644 megaradrp/tests/create.py diff --git a/megaradrp/core/__init__.py b/megaradrp/core/__init__.py new file mode 100644 index 00000000..8d1c0b4c --- /dev/null +++ b/megaradrp/core/__init__.py @@ -0,0 +1,3 @@ + +from .recipe import MegaraBaseRecipe +from .processing import apextract, apextract_tracemap \ No newline at end of file diff --git a/megaradrp/core.py b/megaradrp/core/processing.py similarity index 65% rename from megaradrp/core.py rename to megaradrp/core/processing.py index 899949a1..3ef8c6fe 100644 --- a/megaradrp/core.py +++ b/megaradrp/core/processing.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -22,7 +22,6 @@ from astropy.io import fits import numpy as np -from numina.core import BaseRecipeAutoQC as MegaraBaseRecipe from numina.array.trace.extract import extract_simple_rss from numina.array.utils import wc_to_pix_1d as wcs_to_pix_1d @@ -31,82 +30,8 @@ _direc = ['normal', 'mirror'] -def create(image, direction='normal', bins='11'): - '''Create a image with overscan for testing.''' - - if direction not in _direc: - raise ValueError("%s must be either 'normal' or 'mirror'" % direction) - - if direction == 'normal': - direcfun = lambda x: x - else: - direcfun = np.fliplr - - if bins not in _binning: - raise ValueError("%s must be one if '11', '12', '21, '22'" % bins) - - bng = _binning[bins] - - nr = 2056 / bng[0] - nc = 2048 / bng[1] - - nr2 = 2 * nr - nc2 = 2 * nc - - oscan1 = 50 / bng[0] - oscan2 = oscan1 * 2 - - psc1 = 50 / bng[0] - psc2 = 2 * psc1 - - fshape = (nr2 + oscan2, nc2 + psc2) - - # Row block 1 - rb1 = slice(0, nr) - rb1m = slice(nr, nr + oscan1) - # Row block 2 - rb2 = slice(nr + oscan2, nr2 + oscan2) - rb2m = slice(nr + oscan1, nr + oscan2) - # Col block - cb = slice(psc1, nc2 + psc1) - # Col block left - cbl = slice(0, psc1) - # Col block right - cbr = slice(nc2 + psc1, nc2 + psc2) - - # Mode normal - trim1 = (rb1, cb) - pcol1 = (rb1, cbl) - ocol1 = (rb1, cbr) - orow1 = (rb1m, cb) - print(trim1, ocol1, orow1, pcol1) - - trim2 = (rb2, cb) - pcol2 = (rb2, cbr) - ocol2 = (rb2, cbl) - orow2 = (rb2m, cb) - print(trim2, ocol2, orow2, pcol2) - - finaldata = np.zeros(fshape, dtype='float32') - - finaldata[trim1] = direcfun(np.atleast_2d(np.arange(0, nc2))) - finaldata[trim2] = direcfun(np.atleast_2d(np.arange(0, nc2))) - - finaldata[orow1] = 3 - finaldata[orow2] = 4 - - finaldata[pcol1] = 5 - finaldata[pcol2] = 6 - - finaldata[ocol1] = 7 - finaldata[ocol2] = 8 - - hdu = fits.PrimaryHDU(data=finaldata) - hdu.writeto(image, clobber=True) - - def trim_and_o(image, out='trimmed.fits', direction='normal', bins='11'): - '''Trim a MEGARA image with overscan.''' + """Trim a MEGARA image with overscan.""" with fits.open(image) as hdul: hdu = trim_and_o_hdu(hdul[0]) @@ -114,7 +39,7 @@ def trim_and_o(image, out='trimmed.fits', direction='normal', bins='11'): def trim_and_o_hdu(hdu): - '''Trim a MEGARA HDU with overscan.''' + """Trim a MEGARA HDU with overscan.""" # FIXME: this should come from the header direction = 'normal' @@ -127,7 +52,7 @@ def trim_and_o_hdu(hdu): def trim_and_o_array(array, direction='normal', bins='11'): - '''Trim a MEGARA array with overscan.''' + """Trim a MEGARA array with overscan.""" if direction not in _direc: raise ValueError("%s must be either 'normal' or 'mirror'" % direction) @@ -160,8 +85,9 @@ def trim_and_o_array(array, direction='normal', bins='11'): finaldata[nr:, :] = direcfun(array[nr + oscan2:, psc1:nc2 + psc1]) return finaldata + def apextract(data, trace): - '''Extract apertures.''' + """Extract apertures.""" rss = np.empty((trace.shape[0], data.shape[1]), dtype='float32') for idx, r in enumerate(trace): l = r[0] @@ -191,7 +117,7 @@ def extract_region(data, border1, border2, pesos, xpos): for x, a,b in zip(xpos, border1, border2): fill_other(pesos[:,x], a, b) - final2d = data[region,:] * pesos[region,:] + final2d = data[region,:] * pesos[region,:] pesos[region,:] = 0.0 final = final2d.sum(axis=0) @@ -199,7 +125,7 @@ def extract_region(data, border1, border2, pesos, xpos): def apextract_tracemap(data, tracemap): - '''Extract apertures using a tracemap.''' + """Extract apertures using a tracemap.""" # FIXME: a little hackish @@ -227,7 +153,6 @@ def apextract_tracemap(data, tracemap): rss = extract_simple_rss(data, borders) - return rss diff --git a/megaradrp/core/recipe.py b/megaradrp/core/recipe.py new file mode 100644 index 00000000..7c356e3f --- /dev/null +++ b/megaradrp/core/recipe.py @@ -0,0 +1,31 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + + +from numina.core import BaseRecipe +from numina.core.dataholders import Product +from numina.core.products import QualityControlProduct + + +class MegaraBaseRecipe(BaseRecipe): + """Base clase for all MEGARA Recipes""" + + qc = Product(QualityControlProduct, dest='qc') + + diff --git a/megaradrp/processing/aperture.py b/megaradrp/processing/aperture.py index ac497cc6..54397137 100644 --- a/megaradrp/processing/aperture.py +++ b/megaradrp/processing/aperture.py @@ -19,18 +19,16 @@ import logging - from numina.flow.processing import TagOptionalCorrector, TagFits -from ..core import apextract, apextract_tracemap +from megaradrp.core.processing import apextract, apextract_tracemap _logger = logging.getLogger('numina.processing') class ApertureExtractor(TagOptionalCorrector): - - '''A Node that extracts apertures.''' + """A Node that extracts apertures.""" def __init__(self, trace, datamodel=None, mark=True, tagger=None, dtype='float32'): @@ -54,8 +52,7 @@ def _run(self, img): class ApertureExtractor2(TagOptionalCorrector): - - '''A Node that extracts apertures.''' + """A Node that extracts apertures.""" def __init__(self, trace, datamodel=None, mark=True, tagger=None, dtype='float32'): diff --git a/megaradrp/processing/trimover.py b/megaradrp/processing/trimover.py index c0849d21..10cf0f7c 100644 --- a/megaradrp/processing/trimover.py +++ b/megaradrp/processing/trimover.py @@ -21,7 +21,7 @@ from numina.flow.processing import TagOptionalCorrector, TagFits -from ..core import trim_and_o_hdu +from ..core.processing import trim_and_o_hdu _logger = logging.getLogger('megara.processing') diff --git a/megaradrp/tests/create.py b/megaradrp/tests/create.py new file mode 100644 index 00000000..2782122b --- /dev/null +++ b/megaradrp/tests/create.py @@ -0,0 +1,101 @@ +# +# Copyright 2011-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + +from __future__ import print_function + +from astropy.io import fits +import numpy as np + +# row / column +_binning = {'11': [1, 1], '21': [1, 2], '12': [2, 1], '22': [2, 2]} +_direc = ['normal', 'mirror'] + + +def create(image, direction='normal', bins='11'): + """Create a image with overscan for testing.""" + + if direction not in _direc: + raise ValueError("%s must be either 'normal' or 'mirror'" % direction) + + if direction == 'normal': + direcfun = lambda x: x + else: + direcfun = np.fliplr + + if bins not in _binning: + raise ValueError("%s must be one if '11', '12', '21, '22'" % bins) + + bng = _binning[bins] + + nr = 2056 / bng[0] + nc = 2048 / bng[1] + + nr2 = 2 * nr + nc2 = 2 * nc + + oscan1 = 50 / bng[0] + oscan2 = oscan1 * 2 + + psc1 = 50 / bng[0] + psc2 = 2 * psc1 + + fshape = (nr2 + oscan2, nc2 + psc2) + + # Row block 1 + rb1 = slice(0, nr) + rb1m = slice(nr, nr + oscan1) + # Row block 2 + rb2 = slice(nr + oscan2, nr2 + oscan2) + rb2m = slice(nr + oscan1, nr + oscan2) + # Col block + cb = slice(psc1, nc2 + psc1) + # Col block left + cbl = slice(0, psc1) + # Col block right + cbr = slice(nc2 + psc1, nc2 + psc2) + + # Mode normal + trim1 = (rb1, cb) + pcol1 = (rb1, cbl) + ocol1 = (rb1, cbr) + orow1 = (rb1m, cb) + print(trim1, ocol1, orow1, pcol1) + + trim2 = (rb2, cb) + pcol2 = (rb2, cbr) + ocol2 = (rb2, cbl) + orow2 = (rb2m, cb) + print(trim2, ocol2, orow2, pcol2) + + finaldata = np.zeros(fshape, dtype='float32') + + finaldata[trim1] = direcfun(np.atleast_2d(np.arange(0, nc2))) + finaldata[trim2] = direcfun(np.atleast_2d(np.arange(0, nc2))) + + finaldata[orow1] = 3 + finaldata[orow2] = 4 + + finaldata[pcol1] = 5 + finaldata[pcol2] = 6 + + finaldata[ocol1] = 7 + finaldata[ocol2] = 8 + + hdu = fits.PrimaryHDU(data=finaldata) + hdu.writeto(image, clobber=True) From 0ce7b5984a9665562567e2ee0a18ebb4b7ca5a91 Mon Sep 17 00:00:00 2001 From: Pica Date: Thu, 19 Nov 2015 18:22:37 +0100 Subject: [PATCH 7/8] Extend MegaraBaseRecipe --- megaradrp/core/recipe.py | 72 +++++++++++++++++++++++++++++ megaradrp/core/tests/__init__.py | 0 megaradrp/core/tests/test_recipe.py | 12 +++++ 3 files changed, 84 insertions(+) create mode 100644 megaradrp/core/tests/__init__.py create mode 100644 megaradrp/core/tests/test_recipe.py diff --git a/megaradrp/core/recipe.py b/megaradrp/core/recipe.py index 7c356e3f..e4c3946e 100644 --- a/megaradrp/core/recipe.py +++ b/megaradrp/core/recipe.py @@ -17,15 +17,87 @@ # along with Megara DRP. If not, see . # +import logging +from astropy.io import fits + +import numina.array.combine as combine +from numina.flow import SerialFlow +from numina.flow.processing import BiasCorrector from numina.core import BaseRecipe from numina.core.dataholders import Product from numina.core.products import QualityControlProduct +from megaradrp.processing import OverscanCorrector, TrimImage + +_logger = logging.getLogger('numina.recipes.megara') class MegaraBaseRecipe(BaseRecipe): """Base clase for all MEGARA Recipes""" qc = Product(QualityControlProduct, dest='qc') + def __init__(self, version): + self.__flow = {'BiasRecipe':[OverscanCorrector, TrimImage], + 'ArcCalibrationRecipe':[OverscanCorrector, TrimImage, BiasCorrector], + 'FiberFlatRecipe':[OverscanCorrector, TrimImage, BiasCorrector], + 'TraceMapRecipe':[OverscanCorrector, TrimImage, BiasCorrector], + } + super(MegaraBaseRecipe, self).__init__(version=version) + + def __generate_flow(self, params): + flow = self.__flow[self.__class__.__name__] + try: + for cont in range(len(flow)): + if issubclass(BiasCorrector, flow[cont]): + flow[cont] = (flow[cont](params['biasmap'])) + elif issubclass(TrimImage, flow[cont]) or issubclass(OverscanCorrector, flow[cont]): + flow[cont] = (flow[cont]()) + basicflow = SerialFlow(flow) + + except Exception as e: + _logger.error(e) + raise(e) + + return basicflow + + def bias_process_common(self, obresult, master_bias): + + with master_bias.open() as hdul: + mbias = hdul[0].data.copy() + + hdu, data = self.hdu_creation(obresult, {'biasmap':mbias}) + + hdr = hdu.header + # FIXME: this is incorrect in general + hdr['IMGTYP'] = ('FIBER_FLAT', 'Image type') + hdr['NUMTYP'] = ('MASTER_FIBER_FLAT', 'Data product type') + hdr = self.set_base_headers(hdr) + hdr['CCDMEAN'] = data[0].mean() + + varhdu = fits.ImageHDU(data[1], name='VARIANCE') + num = fits.ImageHDU(data[2], name='MAP') + reduced = fits.HDUList([hdu, varhdu, num]) + return reduced + + def hdu_creation(self, obresult, params=None): + + basicflow = self.__generate_flow(params) + + cdata = [] + try: + for frame in obresult.images: + hdulist = frame.open() + hdulist = basicflow(hdulist) + cdata.append(hdulist) + + _logger.info('stacking %d images using median', len(cdata)) + + data = combine.median([d[0].data for d in cdata], dtype='float32') + template_header = cdata[0][0].header + hdu = fits.PrimaryHDU(data[0], header=template_header) + finally: + for hdulist in cdata: + hdulist.close() + return hdu, data \ No newline at end of file diff --git a/megaradrp/core/tests/__init__.py b/megaradrp/core/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megaradrp/core/tests/test_recipe.py b/megaradrp/core/tests/test_recipe.py new file mode 100644 index 00000000..75b2f9c7 --- /dev/null +++ b/megaradrp/core/tests/test_recipe.py @@ -0,0 +1,12 @@ + + +from numina.core import BaseRecipe + +from megaradrp.core.recipe import MegaraBaseRecipe + + +def test_base_recipe(): + version = "1.0.1" + obj = MegaraBaseRecipe(version) + assert isinstance(obj, BaseRecipe) + assert obj.__version__ == version \ No newline at end of file From 213009135e5bb66a2030aec765a0c6e059d7ae78 Mon Sep 17 00:00:00 2001 From: Pica Date: Thu, 19 Nov 2015 18:35:28 +0100 Subject: [PATCH 8/8] Recipes have been isolated --- megaradrp/recipes/calibration/arc.py | 120 +++++++------------------ megaradrp/recipes/calibration/base.py | 2 +- megaradrp/recipes/calibration/bias.py | 58 ++++++------ megaradrp/recipes/calibration/flat.py | 78 +++------------- megaradrp/recipes/calibration/trace.py | 34 ++++--- 5 files changed, 95 insertions(+), 197 deletions(-) diff --git a/megaradrp/recipes/calibration/arc.py b/megaradrp/recipes/calibration/arc.py index b2eefd64..21818ddf 100644 --- a/megaradrp/recipes/calibration/arc.py +++ b/megaradrp/recipes/calibration/arc.py @@ -17,7 +17,7 @@ # along with Megara DRP. If not, see . # -'''Calibration Recipes for Megara''' +"""Arc Calibration Recipe for Megara""" from __future__ import division, print_function @@ -25,39 +25,31 @@ import numpy from astropy.io import fits +from scipy.interpolate import interp1d from numina.core import Requirement, Product, Parameter from numina.core import DataFrameType from numina.core.products import ArrayType from numina.core.requirements import ObservationResultRequirement -from numina.array.combine import median as c_median -from numina.flow import SerialFlow -from numina.flow.processing import BiasCorrector - -# For WL calibration -# FIXME: remove this later from numina.core.products import LinesCatalog -from scipy.interpolate import interp1d from numina.array.wavecal.arccalibration import arccalibration_direct from numina.array.wavecal.arccalibration import fit_solution from numina.array.wavecal.arccalibration import gen_triplets_master from numina.array.wavecal.statsummary import sigmaG +# FIXME: still using findpeaks1D functions from numina.array.peaks.findpeaks1D import findPeaks_spectrum from numina.array.peaks.findpeaks1D import refinePeaks_spectrum from megaradrp.core import MegaraBaseRecipe -from megaradrp.processing import OverscanCorrector, TrimImage - from megaradrp.products import TraceMap from megaradrp.requirements import MasterBiasRequirement - -from megaradrp.core import apextract_tracemap +from megaradrp.core.processing import apextract_tracemap _logger = logging.getLogger('numina.recipes.megara') class ArcCalibrationRecipe(MegaraBaseRecipe): - '''Process ARC images and create WL_CALIBRATION.''' + """Process ARC images and create WL_CALIBRATION.""" # Requirements obresult = ObservationResultRequirement() @@ -71,13 +63,11 @@ class ArcCalibrationRecipe(MegaraBaseRecipe): wlcalib = Product(ArrayType) def __init__(self): - super(ArcCalibrationRecipe, self).__init__( - version="0.1.0" - ) + super(ArcCalibrationRecipe, self).__init__("0.1.0") def run(self, rinput): # Basic processing - reduced = self.process_common(rinput.obresult, rinput.master_bias) + reduced = self.bias_process_common(rinput.obresult, rinput.master_bias) _logger.info('extract fibers') rssdata = apextract_tracemap(reduced[0].data, rinput.tracemap) @@ -93,12 +83,12 @@ def run(self, rinput): wlcalib=coeff_table) def calibrate_wl(self, rss, lines_catalog, poldeg, times_sigma=50.0): - # + # # read master table (TBM) and generate auxiliary parameters (valid for # all the slits) for the wavelength calibration - wv_master = lines_catalog[:,0] + wv_master = lines_catalog[:, 0] ntriplets_master, ratios_master_sorted, triplets_master_sorted_list = \ - gen_triplets_master(wv_master) + gen_triplets_master(wv_master) # FIXME: this depends on the spectral and dispersion axes nspec = rss.shape[0] coeff_table = numpy.zeros((nspec, poldeg + 1)) @@ -107,19 +97,16 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, times_sigma=50.0): for idx, row in enumerate(rss): _logger.info('Starting row %d', idx) # find peaks (initial search providing integer numbers) - threshold = numpy.median(row)+times_sigma*sigmaG(row) - ipeaks_int = findPeaks_spectrum(row, nwinwidth=nwinwidth, - data_threshold=threshold) - # refine peaks fitting an appropriate function (providing float + threshold = numpy.median(row) + times_sigma * sigmaG(row) + ipeaks_int = findPeaks_spectrum(row, nwinwidth, threshold) + # refine peaks fitting an appropriate function (providing float # numbers) - ipeaks_float = refinePeaks_spectrum(row, ipeaks_int, nwinwidth, - method=2) - - - # define interpolation function and interpolate the refined peak - # location, passing from index number (within the row array) - # to channel number (note that this step takes care of the fact - # that the extracted spectrum may correspond to a subregion in the + ipeaks_float = refinePeaks_spectrum(row, ipeaks_int, nwinwidth) + + # define interpolation function and interpolate the refined peak + # location, passing from index number (within the row array) + # to channel number (note that this step takes care of the fact + # that the extracted spectrum may correspond to a subregion in the # spectral direction) # FIXME: xchannel ??? @@ -128,7 +115,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, times_sigma=50.0): naxis1 = row.shape[0] xchannel = numpy.arange(1, naxis1 + 1) - finterp_channel = interp1d(range(xchannel.size), xchannel, + finterp_channel = interp1d(range(xchannel.size), xchannel, kind='linear') xpeaks_refined = finterp_channel(ipeaks_float) @@ -139,7 +126,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, times_sigma=50.0): triplets_master_sorted_list, xpeaks_refined, naxis1, - wv_ini_search=3500, + wv_ini_search=3500, wv_end_search=4500, error_xpos_arc=2.0, times_sigma_r=3.0, @@ -148,64 +135,25 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, times_sigma=50.0): poly_degree_wfit=2, times_sigma_polfilt=10.0, times_sigma_inclusion=5.0) + _logger.info('Solution for row %d completed', idx) _logger.info('Fitting solution for row %d', idx) numpy_array_with_coeff, crval1_approx, cdelt1_approx = \ - fit_solution(wv_master, - xpeaks_refined, - solution, - poly_degree_wfit=2, - weighted=False) - + fit_solution(wv_master, + xpeaks_refined, + solution, + poly_degree_wfit=2, + weighted=False) + _logger.info('approximate crval1, cdelt1: %f %f', - crval1_approx,cdelt1_approx) - _logger.info('fitted coefficients %s',numpy_array_with_coeff) + crval1_approx, + cdelt1_approx) + + _logger.info('fitted coefficients %s', numpy_array_with_coeff) + coeff_table[idx] = numpy_array_with_coeff + except TypeError as error: _logger.error("%s", error) return coeff_table - - - def process_common(self, obresult, master_bias): - _logger.info('starting prereduction') - - o_c = OverscanCorrector() - t_i = TrimImage() - - with master_bias.open() as hdul: - mbias = hdul[0].data.copy() - b_c = BiasCorrector(mbias) - - basicflow = SerialFlow([o_c, t_i, b_c]) - - cdata = [] - - try: - for frame in obresult.images: - hdulist = frame.open() - hdulist = basicflow(hdulist) - cdata.append(hdulist) - - _logger.info('stacking %d images using median', len(cdata)) - - data = c_median([d[0].data for d in cdata], dtype='float32') - template_header = cdata[0][0].header - hdu = fits.PrimaryHDU(data[0], header=template_header) - finally: - for hdulist in cdata: - hdulist.close() - - hdr = hdu.header - hdr['IMGTYP'] = ('FIBER_FLAT', 'Image type') - hdr['NUMTYP'] = ('MASTER_FIBER_FLAT', 'Data product type') - hdr = self.set_base_headers(hdr) - hdr['CCDMEAN'] = data[0].mean() - - varhdu = fits.ImageHDU(data[1], name='VARIANCE') - num = fits.ImageHDU(data[2], name='MAP') - result = fits.HDUList([hdu, varhdu, num]) - - _logger.info('prereduction ended') - - return result diff --git a/megaradrp/recipes/calibration/base.py b/megaradrp/recipes/calibration/base.py index 4f4f7111..e1f56d05 100644 --- a/megaradrp/recipes/calibration/base.py +++ b/megaradrp/recipes/calibration/base.py @@ -97,7 +97,7 @@ class S_And_E_FromStdStarsRecipe(MegaraBaseRecipe): traces = Product(ArrayType) def __init__(self): - super(SensitivityFromStdStarRecipe, self).__init__( + super(S_And_E_FromStdStarsRecipe, self).__init__( version="0.1.0" ) diff --git a/megaradrp/recipes/calibration/bias.py b/megaradrp/recipes/calibration/bias.py index 386a98c7..fb411813 100644 --- a/megaradrp/recipes/calibration/bias.py +++ b/megaradrp/recipes/calibration/bias.py @@ -1,21 +1,37 @@ +# +# Copyright 2014-2015 Universidad Complutense de Madrid +# +# This file is part of Megara DRP +# +# Megara DRP is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Megara DRP is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Megara DRP. If not, see . +# + import logging from astropy.io import fits -from megaradrp.core import MegaraBaseRecipe -from megaradrp.processing import OverscanCorrector, TrimImage -from megaradrp.products import MasterBias - -from numina.array.combine import median as c_median from numina.core import Product, RecipeError from numina.core.requirements import ObservationResultRequirement -from numina.flow import SerialFlow + +from megaradrp.core import MegaraBaseRecipe +from megaradrp.products import MasterBias _logger = logging.getLogger('numina.recipes.megara') class BiasRecipe(MegaraBaseRecipe): - '''Process BIAS images and create MASTER_BIAS.''' + """Process BIAS images and create MASTER_BIAS.""" obresult = ObservationResultRequirement() biasframe = Product(MasterBias) @@ -24,33 +40,13 @@ def __init__(self): super(BiasRecipe, self).__init__(version="0.1.0") def run(self, rinput): - return self.process(rinput.obresult) - def process(self, obresult): _logger.info('starting bias reduction') - if not obresult.images: + if not rinput.obresult.images: raise RecipeError('Frame list is empty') - cdata = [] - o_c = OverscanCorrector() - t_i = TrimImage() - basicflow = SerialFlow([o_c, t_i]) - - try: - for frame in obresult.images: - hdulist = frame.open() - hdulist = basicflow(hdulist) - cdata.append(hdulist) - - _logger.info('stacking %d images using median', len(cdata)) - - data = c_median([d[0].data for d in cdata], dtype='float32') - template_header = cdata[0][0].header - hdu = fits.PrimaryHDU(data[0], header=template_header) - finally: - for hdulist in cdata: - hdulist.close() + hdu, data = self.hdu_creation(rinput.obresult) hdr = hdu.header hdr = self.set_base_headers(hdr) @@ -63,5 +59,5 @@ def process(self, obresult): hdulist = fits.HDUList([hdu, varhdu, num]) _logger.info('bias reduction ended') - result = self.create_result(biasframe=hdu) - return result \ No newline at end of file + result = self.create_result(biasframe=hdulist) + return result diff --git a/megaradrp/recipes/calibration/flat.py b/megaradrp/recipes/calibration/flat.py index 45189eeb..c621e219 100644 --- a/megaradrp/recipes/calibration/flat.py +++ b/megaradrp/recipes/calibration/flat.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2014 Universidad Complutense de Madrid +# Copyright 2011-2015 Universidad Complutense de Madrid # # This file is part of Megara DRP # @@ -17,7 +17,7 @@ # along with Megara DRP. If not, see . # -"""Calibration Recipes for Megara""" +"""Fiber flat calibration Recipe for Megara""" from __future__ import division, print_function @@ -27,12 +27,8 @@ from numina.core import Product, Requirement from numina.core.requirements import ObservationResultRequirement -from numina.array.combine import median as c_median -from numina.flow import SerialFlow -from numina.flow.processing import BiasCorrector from megaradrp.core import MegaraBaseRecipe -from megaradrp.processing import OverscanCorrector, TrimImage from megaradrp.products import MasterFiberFlat from megaradrp.products import TraceMap from megaradrp.requirements import MasterBiasRequirement @@ -41,83 +37,35 @@ _logger = logging.getLogger('numina.recipes.megara') -def process_common(recipe, obresult, master_bias): - _logger.info('starting prereduction') - - o_c = OverscanCorrector() - t_i = TrimImage() - - with master_bias.open() as hdul: - mbias = hdul[0].data.copy() - b_c = BiasCorrector(mbias) - - basicflow = SerialFlow([o_c, t_i, b_c]) - - cdata = [] - - try: - for frame in obresult.images: - hdulist = frame.open() - hdulist = basicflow(hdulist) - cdata.append(hdulist) - - _logger.info('stacking %d images using median', len(cdata)) - - data = c_median([d[0].data for d in cdata], dtype='float32') - template_header = cdata[0][0].header - hdu = fits.PrimaryHDU(data[0], header=template_header) - finally: - for hdulist in cdata: - hdulist.close() - - hdr = hdu.header - hdr['IMGTYP'] = ('FIBER_FLAT', 'Image type') - hdr['NUMTYP'] = ('MASTER_FIBER_FLAT', 'Data product type') - hdr = recipe.set_base_headers(hdr) - hdr['CCDMEAN'] = data[0].mean() - - varhdu = fits.ImageHDU(data[1], name='VARIANCE') - num = fits.ImageHDU(data[2], name='MAP') - result = fits.HDUList([hdu, varhdu, num]) - - _logger.info('prereduction ended') - - return result - - class FiberFlatRecipe(MegaraBaseRecipe): - '''Process FIBER_FLAT images and create MASTER_FIBER_FLAT.''' + """Process FIBER_FLAT images and create MASTER_FIBER_FLAT.""" # Requirements - master_bias = MasterBiasRequirement() obresult = ObservationResultRequirement() + master_bias = MasterBiasRequirement() + tracemap = Requirement(TraceMap, 'Trace information of the Apertures') # Products fiberflat_frame = Product(MasterFiberFlat) fiberflat_rss = Product(MasterFiberFlat) def __init__(self): - super(FiberFlatRecipe, self).__init__( - version="0.1.0" - ) + super(FiberFlatRecipe, self).__init__(version="0.1.0") def run(self, rinput): - _logger.info('starting fiber flat reduction') + # Basic processing + reduced = self.bias_process_common(rinput.obresult, rinput.master_bias) - reduced = process_common(self, rinput.obresult, rinput.master_bias) - - _logger.info('extract fibers') + _logger.info('extract fibers') rssdata = apextract_tracemap(reduced[0].data, rinput.tracemap) # FIXME: we are ignoring here all the possible bad pixels # and WL distortion when doing the normalization - rssdata /= rssdata.mean() + # rssdata /= rssdata.mean() #Originally uncomment rsshdu = fits.PrimaryHDU(rssdata, header=reduced[0].header) rss = fits.HDUList([rsshdu]) + _logger.info('extraction completed') - _logger.info('fiber flat reduction ended') - result = self.create_result(fiberflat_frame=reduced, - fiberflat_rss=rss) - - return result + result = self.create_result(fiberflat_frame=reduced, fiberflat_rss=rss) + return result \ No newline at end of file diff --git a/megaradrp/recipes/calibration/trace.py b/megaradrp/recipes/calibration/trace.py index 1b8eadf0..ff011633 100644 --- a/megaradrp/recipes/calibration/trace.py +++ b/megaradrp/recipes/calibration/trace.py @@ -24,16 +24,17 @@ import logging import numpy +from astropy.io import fits + from numina.array.trace.traces import trace from numina.core import Product from numina.core.requirements import ObservationResultRequirement +from megaradrp.core import apextract_tracemap +from megaradrp.products import MasterFiberFlat, TraceMap from megaradrp.core import MegaraBaseRecipe -from megaradrp.products import MasterFiberFlat -from megaradrp.products import TraceMap from megaradrp.requirements import MasterBiasRequirement from megaradrp.trace.traces import init_traces -from .flat import process_common _logger = logging.getLogger('numina.recipes.megara') @@ -52,11 +53,21 @@ def __init__(self): def run(self, rinput): - result = self.process_base(rinput.obresult, rinput.master_bias) + # Basic processing + reduced = self.bias_process_common(rinput.obresult, rinput.master_bias) + + _logger.info('extract fibers') + rssdata = apextract_tracemap(reduced[0].data, rinput.tracemap) + # FIXME: we are ignoring here all the possible bad pixels + # and WL distortion when doing the normalization + # rssdata /= rssdata.mean() #Originally uncomment + rsshdu = fits.PrimaryHDU(rssdata, header=reduced[0].header) + rss = fits.HDUList([rsshdu]) - data = result[0].data + _logger.info('extraction completed') + _logger.info('fiber flat reduction ended') - # fit_traces = domefun(data, cstart=2000, hs=20) + data = rss[0].data cstart = 2000 hs = 3 @@ -77,7 +88,7 @@ def run(self, rinput): image2 = data.byteswap().newbyteorder() else: image2 = data - + _logger.info('trace peaks') for dtrace in central_peaks.values(): @@ -85,14 +96,9 @@ def run(self, rinput): hs=hs, background=background1, maxdis=maxdis1) pfit = numpy.polyfit(mm[:,0], mm[:,1], deg=5) - + tracelist.append({'fibid': dtrace.fibid, 'boxid': dtrace.boxid, 'start':0, 'stop':4095, 'fitparms': pfit.tolist()}) - return self.create_result(fiberflat_frame=result, - traces=tracelist) - - def process_base(self, obresult, master_bias): - reduced = process_common(self, obresult, master_bias) - return reduced + return self.create_result(fiberflat_frame=rss, traces=tracelist)