WIP lots of fixes for ImageFileCollection

CHANGES.rst

@@ -4,6 +4,12 @@
 New Features
 ^^^^^^^^^^^^
 
+- Allow initialization of ``ImageFileCollection`` from a list of files with no
+  location set. [#374, #661, #680]
+
+- Allow identification of FITS files in ``ImageFileCollection`` based on content
+  of the files instead of file name extension. [#620, #680]
+
 Other Changes and Additions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -19,8 +25,15 @@ Other Changes and Additions
 
 - Added support for .bz2, .Z and .zip file formats in ``ImageFileCollection``.
 
+- Removed support for initializing ``ImageFileCollection`` from a table instead
+  of files. [#680]
+
+- More consistent typing of ``ImageFileCollection.summary`` when the collection
+  is empty. [#601, #680]
+
 Bug Fixes
 ^^^^^^^^^
+
 - Function ``median_combine`` now correctly calculates the uncertainty for
   masked ``CCDData``. [#608]
 
@@ -29,6 +42,9 @@ Bug Fixes
 - Raise ``ValueError`` error in ``subtract_dark`` for when the errors have
   different shapes [#674, #677]
 
+- Fix problem with column dtypes when initializing ``ImageFileCollection`` from
+  a list of file names. [#662, #680]
+
 1.3.0 (2017-11-1)
 -----------------
 

ccdproc/image_collection.py

@@ -48,18 +48,16 @@ class ImageFileCollection(object):
         columns. Default value is '*' unless ``info_file`` is specified.
         Default is ``None``.
 
-    info_file : str or None, optional
-        Path to file that contains a table of information about FITS files.
-        In this case the keywords are set to the names of the columns of the
-        ``info_file`` unless ``keywords`` is explicitly set to a different
-        list.
-        Default is ``None``.
-
-        .. deprecated:: 1.3
+    find_fits_by_reading: bool, optional
+        If ``True``, read each file in location to check whether the file is a
+        FITS file and include it in the collection based on that, rather than
+        by file name. Compressed files, e.g. image.fits.gz, will **NOT** be
+        properly detected. *Will be ignored if `filenames` is not ``None``.*
 
     filenames: str, list of str, or None, optional
         List of the names of FITS files which will be added to the collection.
-        The filenames are assumed to be in ``location``.
+        The filenames may either be in ``location`` or the name can be a
+        relative or absolute path to the file.
         Default is ``None``.
 
     glob_include: str or None, optional
@@ -74,23 +72,20 @@ class ImageFileCollection(object):
         easily select subsets of files in the target directory.
         Default is ``None``.
 
-     ext: str or int, optional
-         The extension from which the header and data will be read in all files.
-         Default is ``0``.
+    ext: str or int, optional
+        The extension from which the header and data will be read in all
+        files.Default is ``0``.
 
     Raises
     ------
     ValueError
         Raised if keywords are set to a combination of '*' and any other
         value.
     """
-    def __init__(self, location=None, keywords=None, info_file=None,
-                 filenames=None, glob_include=None, glob_exclude=None, ext=0):
-
-        if info_file is not None:
-            warnings.warn("The 'info_file' argument is deprecated and will be "
-                          "removed in a future version", DeprecationWarning)
 
+    def __init__(self, location=None, keywords=None,
+                 find_fits_by_reading=False,
+                 filenames=None, glob_include=None, glob_exclude=None, ext=0):
         # Include or exclude files from the collection based on glob pattern
         # matching - has to go above call to _get_files()
         if glob_exclude is not None:
@@ -101,41 +96,24 @@ def __init__(self, location=None, keywords=None, info_file=None,
             glob_include = str(glob_include)
         self._glob_include = glob_include
 
-        self._location = location
+        if location is not None:
+            self._location = location
+        else:
+            self._location = ''
+
+        self._find_fits_by_reading = find_fits_by_reading
+
         self._filenames = filenames
         self._files = []
-        self._info_file = info_file
-        if location:
-            self._files = self._get_files()
+        self._files = self._get_files()
 
         if self._files == []:
             warnings.warn("no FITS files in the collection.",
                           AstropyUserWarning)
         self._summary = {}
         if keywords is None:
-            if info_file is not None:
-                # Default to empty list so that keywords will be populated
-                # from table columns names.
-                keywords = []
-            else:
-                # Otherwise use all keywords.
-                keywords = '*'
-        if info_file is not None:
-            try:
-                info_path = path.join(self.location, info_file)
-            except (AttributeError, TypeError):
-                info_path = info_file
-            try:
-                self._summary = Table.read(info_path, format='ascii',
-                                           delimiter=',')
-                self._summary = Table(self._summary, masked=True)
-            except IOError:
-                if location:
-                    logger.warning('unable to open table file %s, will try '
-                                   'initializing from location instead.',
-                                   info_path)
-                else:
-                    raise
+            # Use all keywords.
+            keywords = '*'
 
         # Used internally to keep track of whether the user asked for all
         # keywords or a specific list. The keywords setter takes care of
@@ -159,11 +137,6 @@ def __repr__(self):
         else:
             kw = "keywords={!r}".format(self.keywords[1:])
 
-        if self._info_file is None:
-            infofile = ''
-        else:
-            infofile = "info_file={!r}".format(self._info_file)
-
         if self.glob_exclude is None:
             glob_exclude = ''
         else:
@@ -184,7 +157,7 @@ def __repr__(self):
         else:
             filenames = "filenames={}".format(self._filenames)
 
-        params = [location, kw, infofile, filenames, glob_include, glob_exclude, ext]
+        params = [location, kw, filenames, glob_include, glob_exclude, ext]
         params = ', '.join([p for p in params if p])
 
         str_repr = "{self.__class__.__name__}({params})".format(
@@ -491,12 +464,19 @@ def _add_val_to_dict(key, value, tbl_dict, n_previous, missing_marker):
 
         assert 'file' not in h
 
+        if self.location:
+            # We have a location and can reconstruct the path using it
+            name_for_file_column = path.basename(file_name)
+        else:
+            # No location, so use whatever path the user passed in
+            name_for_file_column = file_name
+
         # Try opening header before this so that file name is only added if
         # file is valid FITS
         try:
-            summary['file'].append(path.basename(file_name))
+            summary['file'].append(name_for_file_column)
         except KeyError:
-            summary['file'] = [path.basename(file_name)]
+            summary['file'] = [name_for_file_column]
 
         missing_in_this_file = [k for k in summary if (k not in h and
                                                        k != 'file')]
@@ -738,8 +718,16 @@ def _fits_files_in_directory(self, extensions=None,
 
         all_files = listdir(self.location)
         files = []
-        for extension in full_extensions:
-            files.extend(fnmatch.filter(all_files, '*' + extension))
+        if not self._find_fits_by_reading:
+            for extension in full_extensions:
+                files.extend(fnmatch.filter(all_files, '*' + extension))
+        else:
+            for infile in all_files:
+                with open(infile, 'rb') as fp:
+                    # Hmm, first argument to is_fits is not actually used in
+                    # that function. *shrug*
+                    if fits.connect.is_fits('just some junk', infile, fp):
+                        files.append(infile)
 
         files.sort()
         return files