Refactor available datasets logic to be more flexible

satpy/readers/clavrx.py

@@ -102,19 +102,48 @@ def start_time(self):
     def end_time(self):
         return self.filename_info.get('end_time', self.start_time)
 
-    def available_datasets(self):
+    def available_datasets(self, configured_datasets=None):
         """Automatically determine datasets provided by this file"""
         sensor = self.get_sensor(self['/attr/sensor'])
         nadir_resolution = self.get_nadir_resolution(sensor)
+        coordinates = ('longitude', 'latitude')
+        handled_variables = set()
+
+        # update previously configured datasets
+        for is_avail, ds_info in (configured_datasets or []):
+            this_res = ds_info.get('resolution')
+            this_coords = ds_info.get('coordinates')
+            # some other file handler knows how to load this
+            if is_avail is not None:
+                yield is_avail, ds_info
+
+            var_name = ds_info.get('file_key', ds_info['name'])
+            matches = self.file_type_matches(ds_info['file_type'])
+            # we can confidently say that we can provide this dataset and can
+            # provide more info
+            if matches and var_name in self and this_res != nadir_resolution:
+                handled_variables.add(var_name)
+                new_info = ds_info.copy()  # don't mess up the above yielded
+                new_info['resolution'] = nadir_resolution
+                if not self.is_geo and this_coords is None:
+                    new_info['coordinates'] = coordinates
+                yield True, new_info
+            elif is_avail is None:
+                # if we didn't know how to handle this dataset and no one else did
+                # then we should keep it going down the chain
+                yield is_avail, ds_info
+
+        # add new datasets
         for var_name, val in self.file_content.items():
             if isinstance(val, SDS):
                 ds_info = {
                     'file_type': self.filetype_info['file_type'],
                     'resolution': nadir_resolution,
+                    'name': var_name,
                 }
                 if self._is_polar():
                     ds_info['coordinates'] = ['longitude', 'latitude']
-                yield DatasetID(name=var_name, resolution=nadir_resolution), ds_info
+                yield True, ds_info
 
     def get_shape(self, dataset_id, ds_info):
         var_name = ds_info.get('file_key', dataset_id.name)

satpy/readers/file_handlers.py

@@ -122,15 +122,112 @@ def sensor_names(self):
         """List of sensors represented in this file."""
         raise NotImplementedError
 
-    def available_datasets(self):
-        """Get information of available datasets in file.
+    def file_type_matches(self, ds_ftype):
+        """This file handler's type can handle this dataset's file type.
 
-        This is used for dynamically specifying what datasets are available
-        from a file instead of those listed in a YAML configuration file.
+        Args:
+            ds_ftype (str or list): File type or list of file types that a
+                dataset is configured to be loaded from.
 
-        Returns: Iterator of (DatasetID, dict) pairs where dict is the
-                 dataset's metadata, similar to that specified in the YAML
-                 configuration files.
+        Returns: ``True`` if this file handler object's type matches the
+            dataset's file type(s), ``False`` otherwise.
 
         """
-        raise NotImplementedError
+        if isinstance(ds_ftype, str) and ds_ftype == self.filetype_info['file_type']:
+            return True
+        elif self.filetype_info['file_type'] in ds_ftype:
+            return True
+        return None
+
+    def available_datasets(self, configured_datasets=None):
+        """Get information of available datasets in this file.
+
+        This is used for dynamically specifying what datasets are available
+        from a file in addition to what's configured in a YAML configuration
+        file. Note that this method will only be called once for each
+        "file type"; the first file handler for each type.
+
+        This method should **not** update values of the dataset information
+        dictionary **unless* this file handler has a matching file type
+        (the data could be loaded from this object in the future) and at least
+        **one** :class:`satpy.dataset.DatasetID` key is also modified.
+        Otherwise, this file type may override the information provided by
+        a more preferred file type (as specified in the YAML file).
+        It is recommended that any non-ID metadata be updated during the
+        :meth:`BaseFileHandler.get_dataset` part of loading.
+        This method is not guaranteed that it will be called before any
+        other file type's handler.
+        The availability "boolean" not being ``None`` does not mean that a
+        file handler called later can't provide an additional dataset, but
+        it must provide more identifying (DatasetID) information to do so
+        and should yield its new dataset in addition to the previous one.
+
+        Args:
+            configured_datasets (list): Series of (bool, dict) in the same
+                way as is returned by this method (see below). The bool is
+                whether or not the dataset is available from at least one
+                of the current file handlers. It can also be ``None`` if
+                no file handler knows before us knows how to handle it.
+                The dictionary is existing dataset metadata. The dictionaries
+                are typically provided from a YAML configuration file and may
+                be modified, updated, or used as a "template" for additional
+                available datasets. This argument could be the result of a
+                previous file handler's implementation of this method.
+
+        Returns: Iterator of (bool, dict) pairs where dict is the
+            dataset's metadata. If the dataset is available in the current
+            file type then the boolean value should be ``True``, ``False``
+            if we **know** about the dataset but it is unavailable, or
+            ``None`` if this file object is not responsible for it.
+
+        Example 1 - Supplement existing configured information::
+
+            def available_datasets(self, configured_datasets=None):
+                "Add information to configured datasets."
+                # we know the actual resolution
+                res = self.resolution
+
+                # update previously configured datasets
+                for is_avail, ds_info in (configured_datasets or []):
+                    # some other file handler knows how to load this
+                    # don't override what they've done
+                    if is_avail is not None:
+                        yield is_avail, ds_info
+
+                    matches = self.file_type_matches(ds_info['file_type'])
+                    if matches and ds_info.get('resolution') is None:
+                        # we are meant to handle this dataset (file type matches)
+                        # and the information we can provide isn't available yet
+                        new_info = ds_info.copy()
+                        new_info['resolution'] = res
+                        yield True, new_info
+                    elif is_avail is None:
+                        # we don't know what to do with this
+                        # see if another future file handler does
+                        yield is_avail, ds_info
+
+        Example 2 - Add dynamic datasets from the file::
+
+            def available_datasets(self, configured_datasets=None):
+                "Add information to configured datasets."
+                # pass along existing datasets
+                for is_avail, ds_info in (configured_datasets or []):
+                    yield is_avail, ds_info
+
+                # get dynamic variables known to this file (that we created)
+                for var_name, val in self.dynamic_variables.items():
+                    ds_info = {
+                        'file_type': self.filetype_info['file_type'],
+                        'resolution': 1000,
+                        'name': var_name,
+                    }
+                    yield True, ds_info
+
+        """
+        for is_avail, ds_info in (configured_datasets or []):
+            if is_avail is not None:
+                # some other file handler said it has this dataset
+                # we don't know any more information than the previous
+                # file handler so let's yield early
+                yield is_avail, ds_info
+            yield self.file_type_matches(ds_info['file_type']), ds_info

satpy/readers/geocat.py

@@ -129,19 +129,60 @@ def _calc_area_resolution(self, ds_res):
         return self.resolutions.get(sensor, {}).get(int(elem_res),
                                                     elem_res * 1000.)
 
-    def available_datasets(self):
-        """Automatically determine datasets provided by this file"""
+    def available_datasets(self, configured_datasets=None):
+        """Update information for or add datasets provided by this file.
+
+        If this file handler can load a dataset then it will supplement the
+        dataset info with the resolution and possibly coordinate datasets
+        needed to load it. Otherwise it will continue passing the dataset
+        information down the chain.
+
+        See
+        :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets`
+        for details.
+
+        """
         res = self.resolution
-        coordinates = ['pixel_longitude', 'pixel_latitude']
+        coordinates = ('pixel_longitude', 'pixel_latitude')
+        handled_variables = set()
+
+        # update previously configured datasets
+        for is_avail, ds_info in (configured_datasets or []):
+            this_res = ds_info.get('resolution')
+            this_coords = ds_info.get('coordinates')
+            # some other file handler knows how to load this
+            if is_avail is not None:
+                yield is_avail, ds_info
+
+            var_name = ds_info.get('file_key', ds_info['name'])
+            matches = self.file_type_matches(ds_info['file_type'])
+            # we can confidently say that we can provide this dataset and can
+            # provide more info
+            if matches and var_name in self and this_res != res:
+                handled_variables.add(var_name)
+                new_info = ds_info.copy()  # don't mess up the above yielded
+                new_info['resolution'] = res
+                if not self.is_geo and this_coords is None:
+                    new_info['coordinates'] = coordinates
+                yield True, new_info
+            elif is_avail is None:
+                # if we didn't know how to handle this dataset and no one else did
+                # then we should keep it going down the chain
+                yield is_avail, ds_info
+
+        # Provide new datasets
         for var_name, val in self.file_content.items():
+            if var_name in handled_variables:
+                continue
             if isinstance(val, netCDF4.Variable):
                 ds_info = {
                     'file_type': self.filetype_info['file_type'],
                     'resolution': res,
+                    'name': var_name,
                 }
                 if not self.is_geo:
                     ds_info['coordinates'] = coordinates
-                yield DatasetID(name=var_name, resolution=res), ds_info
+                yield True, ds_info
 
     def get_shape(self, dataset_id, ds_info):
         var_name = ds_info.get('file_key', dataset_id.name)

satpy/readers/goes_imager_nc.py

@@ -959,6 +959,33 @@ def __del__(self):
         except (AttributeError, IOError, OSError):
             pass
 
+    def available_datasets(self, configured_datasets=None):
+        """Update information for or add datasets provided by this file.
+
+        If this file handler can load a dataset then it will supplement the
+        dataset info with the resolution and possibly coordinate datasets
+        needed to load it. Otherwise it will continue passing the dataset
+        information down the chain.
+
+        See
+        :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets`
+        for details.
+
+        """
+        res = self.resolution
+        # update previously configured datasets
+        for is_avail, ds_info in (configured_datasets or []):
+            if is_avail is not None:
+                yield is_avail, ds_info
+
+            matches = self.file_type_matches(ds_info['file_type'])
+            if matches and ds_info.get('resolution') is None:
+                new_info = ds_info.copy()
+                new_info['resolution'] = res
+                yield True, new_info
+            elif is_avail is None:
+                yield is_avail, ds_info
+
 
 class GOESNCFileHandler(GOESNCBaseFileHandler):
     """File handler for GOES Imager data in netCDF format"""

satpy/readers/grib.py

@@ -123,9 +123,15 @@ def end_time(self):
         """
         return self._end_time
 
-    def available_datasets(self):
+    def available_datasets(self, configured_datasets=None):
         """Automatically determine datasets provided by this file"""
-        return self._msg_datasets.items()
+        # previously configured or provided datasets
+        # we can't provide any additional information
+        for is_avail, ds_info in (configured_datasets or []):
+            yield is_avail, ds_info
+        # new datasets
+        for ds_info in self._msg_datasets.values():
+            yield True, ds_info
 
     def _get_message(self, ds_info):
         with pygrib.open(self.filename) as grib_file: