Merge 2718fe8 into 7a0e944

.travis.yml

@@ -36,9 +36,9 @@ install:
 - conda info -a
 
 # Create astroconda virtual environment
-- conda env create -q python=$TRAVIS_PYTHON_VERSION -f environment.yml
+- conda env create --force python=$TRAVIS_PYTHON_VERSION -f environment.yml
 - source activate goodman_pipeline
-
+- pip freeze
 # Run DCR installer
 - ./install_dcr.sh
 

environment.yml

@@ -5,7 +5,7 @@ dependencies:
   - numpy
   - pandas==0.23.0
   - matplotlib
-  - scipy
+  - scipy==1.3.1
   - cython
   - astropy
   - pip

goodman_pipeline/core/core.py

@@ -178,6 +178,29 @@ def add_linear_wavelength_solution(ccd, x_axis, reference_lamp, crpix=1):
     return ccd
 
 
+def bias_subtract(ccd, master_bias, master_bias_name):
+    """Subtract bias from file.
+
+    Wrapper for :func:`~ccdproc.subtract_bias`. The main goal is to have a
+    consistent API for apps using the Goodman Pipeline as a library.
+
+    Args:
+        ccd (CCDData): A file to be bias-subtracted
+        master_bias (CCDData):
+        master_bias_name (str): Full path to master bias file, this is added to
+        the bias-subtracted ccd under `GSP_BIAS`.
+
+    Returns:
+        A bias-subtracted file.
+    """
+    ccd = ccdproc.subtract_bias(ccd=ccd, master=master_bias, add_keyword=False)
+    log.info("Bias subtracted")
+    ccd.header.set('GSP_BIAS',
+                   value=os.path.basename(master_bias_name),
+                   comment="Master Bias Image")
+    return ccd
+
+
 def bin_reference_data(wavelength, intensity, serial_binning):
     """Bins a 1D array
 
@@ -1389,7 +1412,7 @@ def get_best_flat(flat_name, path):
     None instead of master_flat_name.
 
     Args:
-        flat_name (str): Full path of master flat basename. Ends in '\*.fits'
+        flat_name (str): Full path of master flat basename. Ends in '*.fits'
           for using glob.
         path (str): Location to look for flats.
 
@@ -3220,7 +3243,7 @@ def write_fits(ccd,
 
     """
     assert isinstance(ccd, CCDData)
-    if not os.path.isdir(os.path.dirname(full_path)):
+    if os.path.isabs(full_path) and not os.path.isdir(os.path.dirname(full_path)):
         log.error("Directory {} does not exist. Creating it right now."
                   "".format(os.path.dirname(full_path)))
         os.mkdir(os.path.dirname(full_path))

goodman_pipeline/core/tests/test_core.py

@@ -35,6 +35,7 @@
 from ..core import (astroscrappy_lacosmic,
                     add_linear_wavelength_solution,
                     add_wcs_keys,
+                    bias_subtract,
                     bin_reference_data,
                     call_cosmic_rejection,
                     classify_spectroscopic_data,
@@ -167,6 +168,29 @@ def test_add_wcs_keys_error(self):
                     'DCLOG1']
 
 
+class BiasSubtractTest(TestCase):
+
+    def setUp(self):
+        self.ccd = CCDData(data=np.ones((100, 100)) * 100,
+                           meta=fits.Header(),
+                           unit='adu')
+        self.master_bias = CCDData(data=np.ones((100, 100)) * 50,
+                           meta=fits.Header(),
+                           unit='adu')
+
+        self.master_bias_name = os.path.join(os.getcwd(),
+                                             'master_bias_file.fits')
+
+    def test_bias_subtract(self):
+
+        ccd = bias_subtract(ccd=self.ccd,
+                            master_bias=self.master_bias,
+                            master_bias_name=self.master_bias_name)
+        np.testing.assert_array_equal(ccd.data, np.ones((100, 100)) * 50.)
+        self.assertEqual(ccd.header['GSP_BIAS'],
+                         os.path.basename(self.master_bias_name))
+
+
 class BinningTest(TestCase):
 
     def test__bin_reference_data(self):
@@ -288,8 +312,8 @@ class CombineDataTest(TestCase):
 
     def setUp(self):
         self.ccd1 = CCDData(data=np.ones((100, 100)),
-                           meta=fits.Header(),
-                           unit='adu')
+                            meta=fits.Header(),
+                            unit='adu')
         self.ccd1.header.set('OBJECT', value='TestObject')
         self.ccd1.header.set('GRATING', value='Grating')
         self.ccd1.header.set('SLIT', value='1.05SlitSize')
@@ -518,7 +542,6 @@ class CreateMasterBias(TestCase):
 
     def setUp(self):
         self.name = ''
-        self.name_2 = ''
         self.bias_files = ['bias_{}.fits'.format(i) for i in range(1, 12)]
         self.raw_data = os.getcwd()
         self.reduced_data = os.getcwd()
@@ -543,8 +566,8 @@ def tearDown(self):
         for _file in self.bias_files:
             os.unlink(os.path.join(self.reduced_data, _file))
 
-        if self.name != '' and self.name_2 != '':
-            for _file in [self.name, self.name_2]:
+        if self.name != '':
+            for _file in [self.name]:
                 os.unlink(_file)
 
     def test_create_master_bias(self):

goodman_pipeline/spectroscopy/tests/test_redspec.py

@@ -2,6 +2,7 @@
 
 import argparse
 import os
+import shutil
 from unittest import TestCase
 
 from ...spectroscopy.redspec import (get_args, MainApp)
@@ -14,8 +15,15 @@ def setUp(self):
         os.mkdir(self.dummy_path)
 
     def tearDown(self):
-        if os.path.exists(self.dummy_path):
-            os.rmdir(self.dummy_path)
+        for _path in [self.dummy_path,
+                      'goodman_pipeline/testing/reference_files',
+                      'goodman_pipeline/testing',
+                      'testing/processed_files',
+                      'testing']:
+            print(_path)
+            if os.path.exists(_path):
+                print("Deleting {}".format(_path))
+                os.rmdir(_path)
 
     def test_input_path_is_relative(self):
         arguments = ['--data-path', self.dummy_path]
@@ -64,6 +72,35 @@ def test_get_args_output_path_does_not_exist(self):
         arguments = ['--data-path', 'does_not_exist']
         self.assertRaises(SystemExit, get_args, arguments)
 
+    def test_reference_dir_does_not_exists(self):
+        arguments = ['--reference-files', 'testing/reference_files']
+
+        args = get_args(arguments=arguments)
+        self.assertTrue(os.path.exists(
+            os.path.join(os.getcwd(),
+                         'goodman_pipeline/testing/reference_files')))
+        self.assertTrue(os.path.isabs(args.reference_dir))
+
+    def test_reference_dir_os_error(self):
+        arguments = ['--reference-files', '/testing/reference_files']
+
+        args = get_args(arguments=arguments)
+        print(args)
+        self.assertFalse(os.path.exists('/testing/reference_files'))
+
+    def test_destination_path_not_abs(self):
+        arguments = ['--proc-path', 'testing/processed_files']
+        args = get_args(arguments=arguments)
+        self.assertTrue(os.path.isabs(args.destination))
+        self.assertTrue(os.path.exists(args.destination))
+        self.assertEqual(args.destination,
+                         os.path.join(os.getcwd(),
+                                      'testing/processed_files'))
+
+    def test_destination_path_sysexit(self):
+        arguments = ['--proc-path', '/testing/processed_files']
+        self.assertRaises(SystemExit, get_args, arguments)
+
 
 def test_get_args():
     from ...spectroscopy.redspec import get_args
@@ -93,8 +130,17 @@ def test_instantiation_without_args(self):
         self.assertIsNone(self.main_app.wavelength_calibration)
         self.assertIsNone(self.main_app.reference)
 
-    def test_main_app(self):
-        pass
+    def test___call___no_args(self):
+        self.assertRaises(SystemExit, self.main_app)
+
+    def test___call___with_valid_arguments(self):
+        arguments = ['--data-path', './',
+                     '--proc-path', './',
+                     '--search-pattern', 'test-pattern',
+                     '--output-prefix', 'w',
+                     '--extraction', 'fractional']
+        args = get_args(arguments=arguments)
+        self.assertRaises(SystemExit, self.main_app, args)
 
 
 if __name__ == '__main__':

goodman_pipeline/spectroscopy/tests/test_wavelength.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import json
 import numpy as np
 import os
 import re
@@ -12,7 +13,8 @@
 from ..wavelength import (WavelengthCalibration)
 
 from ..redspec import get_args
-from ...core import add_wcs_keys
+from ...core import add_wcs_keys, write_fits
+from ...core import ReferenceData, NoMatchFound
 
 
 class WavelengthCalibrationTests(TestCase):
@@ -67,9 +69,15 @@ def tearDown(self):
             if os.path.isfile(_file):
                 os.unlink(_file)
 
-    @skip
-    def test__automatic_wavelength_solution(self):
-        pass
+    # def test__automatic_wavelength_solution_no_match(self):
+    #     self.wc.reference_data = ReferenceData(
+    #         reference_dir='goodman_pipeline/data/ref_comp')
+    #
+    #     self.lamp.header.set('OBJECT', value='PbNa')
+    #     self.wc.lamp = self.lamp
+    #     self.assertRaises(NoMatchFound,
+    #                       self.wc._automatic_wavelength_solution,
+    #                       os.getcwd())
 
     def test__save_wavelength_calibrated(self):
         self.wc.sci_target_file = 'target_sci_file.fits'
@@ -123,7 +131,7 @@ def test___call___method_wrong_ccd(self):
     def test___call___method_wrong_comp_list(self):
         self.assertRaises(AssertionError, self.wc, self.ccd, 'comp_list', '', '')
 
-    def test___call___method_no_comparison_lamps(self):
+    def test___call___method_no_comparison_lamps_json(self):
         json_output = self.wc(ccd=self.ccd,
                               comp_list=[],
                               save_data_to='',
@@ -134,17 +142,78 @@ def test___call___method_no_comparison_lamps(self):
         self.assertEqual(json_output['warning'], '')
         self.assertEqual(json_output['wavelength_solution'], [])
 
+    def test___call___method_no_comparison_lamps(self):
+        output = self.wc(ccd=self.ccd,
+                         comp_list=[],
+                         save_data_to='',
+                         reference_data='',
+                         json_output=False)
+
+        self.assertIsNone(output)
+
     def test___call___method_one_comparison_lamps(self):
         json_output = self.wc(ccd=self.ccd,
                               comp_list=[self.lamp],
                               save_data_to='',
                               reference_data='goodman_pipeline/data/ref_comp',
                               json_output=True)
 
-        print(json_output)
-
         self.assertEqual(json_output['error'], 'Unable to obtain wavelength solution')
         self.assertEqual(json_output['warning'], '')
         self.assertEqual(json_output['wavelength_solution'], [])
 
+    def test___call___method_no_match_found(self):
+        self.lamp.header.set('OBJECT', value='PbNa')
+
+        self.assertRaises(NoMatchFound,
+                          self.wc,
+                          self.ccd,
+                          [self.lamp],
+                          '',
+                          'goodman_pipeline/data/ref_comp')
+
+    def test___call___method_one_solution(self):
+        file_path = os.path.join(os.getcwd(),
+                                 'goodman_pipeline/data/ref_comp',
+                                 'goodman_comp_1200M2_FeHeAr.fits')
+        ref_lamp = CCDData.read(file_path, unit='adu')
+        lamp_ccd = write_fits(ref_lamp, os.path.join(os.getcwd(), 'test_comparison_lamp.fits'), )
+        self.file_list.append('test_comparison_lamp.fits')
+        json_output = self.wc(ccd=self.ccd,
+                              comp_list=[lamp_ccd],
+                              save_data_to='',
+                              reference_data='goodman_pipeline/data/ref_comp',
+                              json_output=True)
+        # print(json.dumps(json_output, indent=4))
+        self.assertEqual(len(json_output['wavelength_solution']), 1)
+        for _solution in json_output['wavelength_solution']:
+            self.file_list.append(_solution['file_name'])
+            self.file_list.append(_solution['reference_lamp'])
+
+    def test___call___method_two_solution(self):
+        file_path_1 = os.path.join(os.getcwd(),
+                                   'goodman_pipeline/data/ref_comp',
+                                   'goodman_comp_1200M2_FeHeAr.fits')
+        file_path_2 = os.path.join(os.getcwd(),
+                                   'goodman_pipeline/data/ref_comp',
+                                   'goodman_comp_1200M2_CuHeAr.fits')
+        ref_lamp_1 = CCDData.read(file_path_1, unit='adu')
+        ref_lamp_2 = CCDData.read(file_path_2, unit='adu')
+        lamp_ccd_1 = write_fits(ref_lamp_1, os.path.join(os.getcwd(), 'test_comparison_lamp_1.fits'), )
+        lamp_ccd_2 = write_fits(ref_lamp_2, os.path.join(os.getcwd(),
+                                                         'test_comparison_lamp_2.fits'), )
+        self.file_list.append('test_comparison_lamp_1.fits')
+        self.file_list.append('test_comparison_lamp_2.fits')
+        json_output = self.wc(ccd=self.ccd,
+                              comp_list=[lamp_ccd_1, lamp_ccd_2],
+                              save_data_to='',
+                              reference_data='goodman_pipeline/data/ref_comp',
+                              json_output=True)
+        # print(json.dumps(json_output, indent=4))
+        self.assertEqual(len(json_output['wavelength_solution']), 2)
+        for _solution in json_output['wavelength_solution']:
+            self.file_list.append(_solution['file_name'])
+            self.file_list.append(_solution['reference_lamp'])
+
+
 

goodman_pipeline/spectroscopy/wavelength.py

@@ -172,11 +172,8 @@ def __call__(self,
             wavelength_solutions = []
             reference_lamp_names = []
             for self.lamp in comp_list:
-                try:
-                    self.calibration_lamp = self.lamp.header['GSP_FNAM']
-                    log.info('Using reference lamp {}'.format(self.calibration_lamp))
-                except KeyError:
-                    self.calibration_lamp = ''
+                self.calibration_lamp = self.lamp.header['GSP_FNAM']
+                log.info('Using reference lamp {}'.format(self.calibration_lamp))
 
                 self.raw_pixel_axis = range(self.lamp.shape[0])
 
@@ -187,9 +184,6 @@ def __call__(self,
 
                 self.lines_center = get_lines_in_lamp(
                     ccd=self.lamp, plots=plots)
-                self.serial_binning, self.parallel_binning = [
-                    int(x) for x in self.lamp.header['CCDSUM'].split()]
-
                 try:
                     self._automatic_wavelength_solution(
                         save_data_to=save_data_to,
@@ -351,6 +345,9 @@ def _automatic_wavelength_solution(self,
         reference_lamp_wav_axis, reference_lamp_ccd.data = \
             self.wcs.read_gsp_wcs(ccd=reference_lamp_ccd)
 
+        self.serial_binning, self.parallel_binning = [
+            int(x) for x in self.lamp.header['CCDSUM'].split()]
+
         if self.serial_binning != 1:
             reference_lamp_wav_axis, reference_lamp_ccd.data = \
                 bin_reference_data(wavelength=reference_lamp_wav_axis,
@@ -436,7 +433,7 @@ def _automatic_wavelength_solution(self,
                           "from {:.3f}".format(correlation_value,
                                                global_cross_corr))
 
-            if plots:
+            if plots:  # pragma: no cover
                 # print(global_cross_corr, correlation_value)
                 plt.ion()
                 plt.title('Samples after cross correlation\n Shift {:.3f}'

goodman_pipeline/version.py

@@ -1,2 +1,2 @@
 # This is an automatic generated file please do not edit
-__version__ = '1.3.0'
+__version__ = '1.3.1.dev1'