Merge pull request #2534 from martin-rdz/feature-viirs-fsspec

Add fsspec functionality to `viirs_sdr` reader

satpy/readers/__init__.py

@@ -20,6 +20,7 @@
 
 import logging
 import os
+import pathlib
 import pickle  # nosec B403
 import warnings
 from datetime import datetime, timedelta
@@ -778,9 +779,20 @@ def _get_compression(file):
 
 
 def open_file_or_filename(unknown_file_thing):
-    """Try to open the *unknown_file_thing*, otherwise return the filename."""
-    try:
-        f_obj = unknown_file_thing.open()
-    except AttributeError:
+    """Try to open the provided file "thing" if needed, otherwise return the filename or Path.
+
+    This wraps the logic of getting something like an fsspec OpenFile object
+    that is not directly supported by most reading libraries and making it
+    usable. If a :class:`pathlib.Path` object or something that is not
+    open-able is provided then that object is passed along. In the case of
+    fsspec OpenFiles their ``.open()`` method is called and the result returned.
+
+    """
+    if isinstance(unknown_file_thing, pathlib.Path):
         f_obj = unknown_file_thing
+    else:
+        try:
+            f_obj = unknown_file_thing.open()
+        except AttributeError:
+            f_obj = unknown_file_thing
     return f_obj

satpy/readers/hdf5_utils.py

@@ -24,6 +24,7 @@
 import numpy as np
 import xarray as xr
 
+from satpy.readers import open_file_or_filename
 from satpy.readers.file_handlers import BaseFileHandler
 from satpy.readers.utils import np2str
 from satpy.utils import get_legacy_chunk_size
@@ -43,7 +44,8 @@ def __init__(self, filename, filename_info, filetype_info):
         self._attrs_cache = {}
 
         try:
-            file_handle = h5py.File(self.filename, "r")
+            f_obj = open_file_or_filename(self.filename)
+            file_handle = h5py.File(f_obj, "r")
         except IOError:
             LOG.exception(
                 "Failed reading file %s. Possibly corrupted file", self.filename)
@@ -73,7 +75,8 @@ def _collect_attrs(self, name, attrs):
 
     def get_reference(self, name, key):
         """Get reference."""
-        with h5py.File(self.filename, "r") as hf:
+        f_obj = open_file_or_filename(self.filename)
+        with h5py.File(f_obj, "r") as hf:
             return self._get_reference(hf, hf[name].attrs[key])
 
     def _get_reference(self, hf, ref):
@@ -97,7 +100,8 @@ def __getitem__(self, key):
         val = self.file_content[key]
         if isinstance(val, h5py.Dataset):
             # these datasets are closed and inaccessible when the file is closed, need to reopen
-            dset = h5py.File(self.filename, "r")[key]
+            f_obj = open_file_or_filename(self.filename)
+            dset = h5py.File(f_obj, "r")[key]
             dset_data = da.from_array(dset, chunks=CHUNK_SIZE)
             attrs = self._attrs_cache.get(key, dset.attrs)
             if dset.ndim == 2: