Merge 5936103 into a2798ef

ccdproc/ccddata.py

@@ -159,9 +159,19 @@ def to_hdu(self):
            hdulist : astropy.io.fits.HDUList object
 
         """
+        from .ccdproc import _short_names
+
         header = fits.Header()
         for k, v in self.header.items():
-            header[k] = v
+            if k in _short_names:
+                # This keyword was (hopefully) added by autologging but the
+                # combination of it and its value FITS-compliant in two ways.
+                # Shorten, sort of...
+                short_name = _short_names[k]
+                header[k] = (short_name, "Shortened name for ccdproc command")
+                header[short_name] = v
+            else:
+                header[k] = v
         hdu = fits.PrimaryHDU(self.data, header)
         hdulist = fits.HDUList([hdu])
         return hdulist

ccdproc/ccdproc.py

@@ -18,6 +18,21 @@
 from .utils.slices import slice_from_string
 from .log_meta import log_to_metadata
 
+# The dictionary below is used to translate actual function names to names
+# that are FITS compliant, i.e. 8 characters or less.
+_short_names = {
+    'background_variance_box': 'bakvarbx',
+    'background_variance_filter': 'bakvfilt',
+    'cosmicray_median': 'crmedian',
+    'create_variance': 'creatvar',
+    'flat_correct': 'flatcor',
+    'gain_correct': 'gaincor',
+    'subtract_bias': 'subbias',
+    'subtract_dark': 'subdark',
+    'subtract_overscan': 'suboscan',
+    'trim_image': 'trimim',
+}
+
 
 @log_to_metadata
 def create_variance(ccd_data, gain=None, readnoise=None):
@@ -590,7 +605,6 @@ def background_variance_filter(data, bbox):
     return ndimage.generic_filter(data, sigma_func, size=(bbox, bbox))
 
 
-@log_to_metadata
 def cosmicray_median(data, thresh,  background=None, mbox=11):
     """
     Identify cosmic rays through median technique.  The median technique

ccdproc/log_meta.py

@@ -4,42 +4,84 @@
 from functools import wraps
 import inspect
 
+import numpy as np
+
 from astropy.extern import six
+from astropy.nddata import NDData
+from astropy import units as u
 
 import ccdproc.ccdproc  # really only need Keyword from ccdproc
 
 _LOG_ARGUMENT = 'add_keyword'
 
 _LOG_ARG_HELP = \
     """
-    {arg} : str, `~ccdproc.ccdproc.Keyword` or dict-like
-        Item(s) to add to metadata of result.
+    {arg} : str, `~ccdproc.ccdproc.Keyword` or dict-like, optional
+        Item(s) to add to metadata of result. Set to None to completely
+        disable logging. Default is to add a dictionary with a single item:
+        the key is the name of this function  and the value is a string
+        containing the arguments the function was called with, except the
+        value of this argument.
     """.format(arg=_LOG_ARGUMENT)
 
 
 def log_to_metadata(func):
     """
     Decorator that adds logging to ccdproc functions
+
+    The decorator adds the optional argument _LOG_ARGUMENT to function
+    signature and updates the function's docstring to reflect that.
+
+    It also sets the default value of the argument to the name of the function
+    and the arguments it was called with.
     """
     func.__doc__ = func.__doc__.format(log=_LOG_ARG_HELP)
+
     (original_args, varargs, keywords, defaults) = inspect.getargspec(func)
+
+    # grab the names of positional arguments for use in automatic logging
+    try:
+        original_positional_args = original_args[:-len(defaults)]
+    except TypeError:
+        original_positional_args = original_args
+
+    # Add logging keyword and its default value for docstring
     original_args.append(_LOG_ARGUMENT)
     try:
         defaults = list(defaults)
     except TypeError:
         defaults = []
     defaults.append(None)
-    original_signature = inspect.formatargspec(original_args, varargs,
-                                               keywords, defaults)
-    original_signature = "{0}{1}".format(func.__name__, original_signature)
-    func.__doc__ = "\n".join([original_signature, func.__doc__])
+
+    signature_with_arg_added = inspect.formatargspec(original_args, varargs,
+                                                     keywords, defaults)
+    signature_with_arg_added = "{0}{1}".format(func.__name__,
+                                               signature_with_arg_added)
+    func.__doc__ = "\n".join([signature_with_arg_added, func.__doc__])
 
     @wraps(func)
     def wrapper(*args, **kwd):
+        # Grab the logging keyword, if it is present.
         log_result = kwd.pop(_LOG_ARGUMENT, False)
         result = func(*args, **kwd)
+
         if log_result:
             _insert_in_metadata(result.meta, log_result)
+        elif log_result is not None:
+            # Logging is not turned off, but user did not provide a value
+            # so construct one.
+            key = func.__name__
+            pos_args = ["{}={}".format(arg_name,
+                                       _replace_array_with_placeholder(arg_value))
+                        for arg_name, arg_value
+                        in zip(original_positional_args, args)]
+            kwd_args = ["{}={}".format(k, _replace_array_with_placeholder(v))
+                        for k, v in six.iteritems(kwd)]
+            pos_args.extend(kwd_args)
+            log_val = ", ".join(pos_args)
+            log_val = log_val.replace("\n", "")
+            to_log = {key: log_val}
+            _insert_in_metadata(result.meta, to_log)
         return result
     return wrapper
 
@@ -56,3 +98,26 @@ def _insert_in_metadata(metadata, arg):
                 metadata[k] = v
         except AttributeError:
             raise
+
+
+def _replace_array_with_placeholder(value):
+    return_type_not_value = False
+    if isinstance(value, u.Quantity):
+        return_type_not_value = not value.isscalar
+    elif isinstance(value, (NDData, np.ndarray)):
+        try:
+            length = len(value)
+        except TypeError:
+            # value has no length...
+            try:
+                # ...but if it is NDData its .data will have a length
+                length = len(value.data)
+            except TypeError:
+                # No idea what this data is, assume length is not 1
+                length = 42
+        return_type_not_value = length > 1
+
+    if return_type_not_value:
+        return "<{}>".format(value.__class__.__name__)
+    else:
+        return value

ccdproc/tests/test_ccddata.py

@@ -121,7 +121,7 @@ def test_header2meta():
 def test_metafromstring_fail():
     hdr = 'this is not a valid header'
     with pytest.raises(TypeError):
-        d1 = CCDData(np.ones((5, 5)), meta=hdr)
+        CCDData(np.ones((5, 5)), meta=hdr)
 
 
 def test_setting_bad_uncertainty_raises_error(ccd_data):
@@ -134,6 +134,34 @@ def test_to_hdu(ccd_data):
     ccd_data.meta = {'observer': 'Edwin Hubble'}
     fits_hdulist = ccd_data.to_hdu()
     assert isinstance(fits_hdulist, fits.HDUList)
+    for k, v in ccd_data.meta.iteritems():
+        assert fits_hdulist[0].header[k] == v
+    np.testing.assert_array_equal(fits_hdulist[0].data, ccd_data.data)
+
+
+def test_to_hdu_long_metadata_item(ccd_data):
+    # There is no attempt to try to handle the general problem of
+    # a long keyword (that requires HIERARCH) with a long string value
+    # (that requires CONTINUE).
+    # However, a long-ish keyword with a long value can happen because of
+    # auto-logging, and we are supposed to handle that.
+
+    # So, a nice long command:
+    from ..ccdproc import subtract_dark, _short_names
+
+    dark = CCDData(np.zeros_like(ccd_data.data), unit="adu")
+    result = subtract_dark(ccd_data, dark, dark_exposure=30 * u.second,
+                           data_exposure=15 * u.second, scale=True)
+    assert 'subtract_dark' in result.header
+    print result.header
+    hdulist = result.to_hdu()
+    header = hdulist[0].header
+    assert header['subtract_dark'] == _short_names['subtract_dark']
+    args_value = header[_short_names['subtract_dark']]
+    # Yuck -- have to hand code the ".0" to the numbers to get this to pass...
+    assert "dark_exposure={0} {1}".format(30.0, u.second) in args_value
+    assert "data_exposure={0} {1}".format(15.0, u.second) in args_value
+    assert "scale=True" in args_value
 
 
 def test_copy(ccd_data):

ccdproc/tests/test_ccdproc.py

@@ -206,14 +206,13 @@ def test_subtract_bias(ccd_data):
     ccd_data.header['key'] = 'value'
     master_bias_array = np.zeros_like(ccd_data.data) + bias_level
     master_bias = CCDData(master_bias_array, unit=ccd_data.unit)
-    no_bias = subtract_bias(ccd_data, master_bias)
+    no_bias = subtract_bias(ccd_data, master_bias, add_keyword=None)
     # Does the data we are left with have the correct average?
     np.testing.assert_almost_equal(no_bias.data.mean(), data_avg)
-    # The test below is *NOT* really the desired outcome. Just here to make
-    # sure a real test gets added when something is done with the metadata.
+    # With logging turned off, metadata should not change
     assert no_bias.header == ccd_data.header
     del no_bias.header['key']
-    assert len(ccd_data.header) > 0
+    assert 'key' in ccd_data.header
     assert no_bias.header is not ccd_data.header
 
 
@@ -245,17 +244,17 @@ def test_subtract_dark(ccd_data, explicit_times, scale, exposure_keyword):
         dark_sub = subtract_dark(ccd_data, master_dark,
                                  dark_exposure=dark_exptime * u.second,
                                  data_exposure=exptime * u.second,
-                                 scale=scale)
+                                 scale=scale, add_keyword=None)
     elif exposure_keyword:
         key = Keyword(exptime_key, unit=u.second)
         dark_sub = subtract_dark(ccd_data, master_dark,
                                  exposure_time=key,
-                                 scale=scale)
+                                 scale=scale, add_keyword=None)
     else:
         dark_sub = subtract_dark(ccd_data, master_dark,
                                  exposure_time=exptime_key,
                                  exposure_unit=u.second,
-                                 scale=scale)
+                                 scale=scale, add_keyword=None)
 
     dark_scale = 1.0
     if scale:

ccdproc/tests/test_ccdproc_logging.py

@@ -1,9 +1,13 @@
+from __future__ import (print_function, division, absolute_import)
+
+import numpy as np
+
 from astropy.extern import six
 from astropy.tests.helper import pytest
 import astropy.units as u
 
-from ..ccdproc import create_variance, Keyword
-
+from ..ccdproc import subtract_bias, create_variance, Keyword
+from ..ccddata import CCDData
 
 @pytest.mark.parametrize('key', [
                          'short',
@@ -54,3 +58,24 @@ def test_log_bad_type_fails(ccd_data):
     with pytest.raises(AttributeError):
         create_variance(ccd_data, readnoise=3 * ccd_data.unit,
                         add_keyword=add_key)
+
+
+def test_log_set_to_None_does_not_change_header(ccd_data):
+    new = create_variance(ccd_data, readnoise=3 * ccd_data.unit,
+                          add_keyword=None)
+    assert new.meta.keys() == ccd_data.header.keys()
+
+
+def test_implicit_logging(ccd_data):
+    # If nothing is supplied for the add_keyword argument then the following
+    # should happen:
+    # + A key named func.__name__ is created, with
+    # + value that is the list of arguments the function was called with.
+    bias = CCDData(np.zeros_like(ccd_data.data), unit="adu")
+    result = subtract_bias(ccd_data, bias)
+    assert "subtract_bias" in result.header
+    assert result.header['subtract_bias'] == "ccd=<CCDData>, master=<CCDData>"
+
+    result = create_variance(ccd_data, readnoise=3 * ccd_data.unit)
+    assert ("readnoise="+str(3 * ccd_data.unit) in
+            result.header['create_variance'])

ccdproc/utils/collections.py

@@ -16,7 +16,7 @@ def __init__(self, *arg, **kwd):
         super(CaseInsensitiveOrderedDict, self).__init__(*arg, **kwd)
 
     def _transform_key(self, key):
-        return key.upper()
+        return key.lower()
 
     def __setitem__(self, key, value):
         super(CaseInsensitiveOrderedDict,