Convert remote files to FSFile objects automatically

continuous_integration/environment.yaml

@@ -44,6 +44,7 @@ dependencies:
   - pytest-cov
   - pytest-lazy-fixture
   - fsspec
+  - s3fs
   - pylibtiff
   - python-geotiepoints
   - pooch

doc/source/index.rst

@@ -56,6 +56,7 @@ the base Satpy installation.
     examples/index
     quickstart
     readers
+    remote_reading
     composites
     resample
     enhancements

doc/source/remote_reading.rst

@@ -0,0 +1,173 @@
+====================
+Reading remote files
+====================
+
+Using a single reader
+=====================
+
+Some of the readers in Satpy can read data directly over various transfer protocols. This is done
+using `fsspec <https://filesystem-spec.readthedocs.io/en/latest/index.html>`_ and various packages
+it is using underneath.
+
+As an example, reading ABI data from public AWS S3 storage can be done in the following way::
+
+    from satpy import Scene
+
+    storage_options = {'anon': True}
+    filenames = ['s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186*']
+    scn = Scene(reader='abi_l1b', filenames=filenames, reader_kwargs={'storage_options': storage_options})
+    scn.load(['true_color_raw'])
+
+Reading from S3 as above requires the `s3fs` library to be installed in addition to `fsspec`.
+
+As an alternative, the storage options can be given using
+`fsspec configuration <https://filesystem-spec.readthedocs.io/en/latest/features.html#configuration>`_.
+For the above example, the configuration could be saved to `s3.json` in the `fsspec` configuration directory
+(by default placed in `~/.config/fsspec/` directory in Linux)::
+
+    {
+        "s3": {
+            "anon": "true"
+        }
+    }
+
+.. note::
+
+    Options given in `reader_kwargs` override only the matching options given in configuration file and everythin else is left
+    as-is. In case of problems in data access, remove the configuration file to see if that solves the issue.
+
+
+For reference, reading SEVIRI HRIT data from a local S3 storage works the same way::
+
+    filenames = [
+        's3://satellite-data-eumetcast-seviri-rss/H-000-MSG3*202204260855*',
+    ]
+    storage_options = {
+        "client_kwargs": {"endpoint_url": "https://PLACE-YOUR-SERVER-URL-HERE"},
+        "secret": "VERYBIGSECRET",
+        "key": "ACCESSKEY"
+    }
+    scn = Scene(reader='seviri_l1b_hrit', filenames=filenames, reader_kwargs={'storage_options': storage_options})
+    scn.load(['WV_073'])
+
+Using the `fsspec` configuration in `s3.json` the configuration would look like this::
+
+    {
+        "s3": {
+            "client_kwargs": {"endpoint_url": "https://PLACE-YOUR-SERVER-URL-HERE"},
+            "secret": "VERYBIGSECRET",
+            "key": "ACCESSKEY"
+        }
+    }
+
+
+Using multiple readers
+======================
+
+If multiple readers are used and the required credentials differ, the storage options are passed per reader like this::
+
+    reader1_filenames = [...]
+    reader2_filenames = [...]
+    filenames = {
+        'reader1': reader1_filenames,
+        'reader2': reader2_filenames,
+    }
+    reader1_storage_options = {...}
+    reader2_storage_options = {...}
+    reader_kwargs = {
+        'reader1': {
+            'option1': 'foo',
+            'storage_options': reader1_storage_options,
+        },
+        'reader2': {
+            'option1': 'foo',
+            'storage_options': reader1_storage_options,
+        }
+    }
+    scn = Scene(filenames=filenames, reader_kwargs=reader_kwargs)
+
+
+Caching the remote files
+========================
+
+Caching the remote file locally can speedup the overall processing time significantly, especially if the data are re-used
+for example when testing. The caching can be done by taking advantage of the `fsspec caching mechanism
+<https://filesystem-spec.readthedocs.io/en/latest/features.html#caching-files-locally>`_::
+
+    reader_kwargs = {
+        'storage_options': {
+            's3': {'anon': True},
+            'simple': {
+                'cache_storage': '/tmp/s3_cache',
+            }
+        }
+    }
+
+    filenames = ['simplecache::s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186*']
+    scn = Scene(reader='abi_l1b', filenames=filenames, reader_kwargs=reader_kwargs)
+    scn.load(['true_color_raw'])
+    scn2 = scn.resample(scn.coarsest_area(), resampler='native')
+    scn2.save_datasets(base_dir='/tmp/', tiled=True, blockxsize=512, blockysize=512, driver='COG', overviews=[])
+
+
+The following table shows the timings for running the above code with different cache statuses::
+
+.. _cache_timing_table:
+
+.. list-table:: Processing times without and with caching
+    :header-rows: 1
+    :widths: 40 30 30
+
+    * - Caching
+      - Elapsed time
+      - Notes
+    * - No caching
+      - 650 s
+      - remove `reader_kwargs` and `simplecache::` from the code
+    * - File cache
+      - 66 s
+      - Initial run
+    * - File cache
+      - 13 s
+      - Second run
+
+.. note::
+
+    The cache is not cleaned by Satpy nor fsspec so the user should handle cleaning excess files from `cache_storage`.
+
+
+.. note::
+
+    Only `simplecache` is considered thread-safe, so using the other caching mechanisms may or may not work depending
+    on the reader, Dask scheduler or the phase of the moon.
+
+
+Resources
+=========
+
+See :class:`~satpy.readers.FSFile` for direct usage of `fsspec` with Satpy, and
+`fsspec documentation <https://filesystem-spec.readthedocs.io/en/latest/index.html>`_ for more details on connection options
+and detailes.
+
+
+Supported readers
+=================
+
+.. _reader_table:
+
+.. list-table:: Satpy Readers capable of reading remote files using `fsspec`
+    :header-rows: 1
+    :widths: 70 30
+
+    * - Description
+      - Reader name
+    * - MSG (Meteosat 8 to 11) SEVIRI data in HRIT format
+      - `seviri_l1b_hrit`
+    * - GOES-R imager data in netcdf format
+      - `abi_l1b`
+    * - NOAA GOES-R ABI L2+ products in netcdf format
+      - `abi_l2_nc`
+    * - Sentinel-3 A and B OLCI Level 1B data in netCDF4 format
+      - `olci_l1b`
+    * - Sentinel-3 A and B OLCI Level 2 data in netCDF4 format
+      - `olci_l2`

satpy/readers/__init__.py

@@ -647,7 +647,10 @@ def _get_reader_kwargs(reader, reader_kwargs):
 class FSFile(os.PathLike):
     """Implementation of a PathLike file object, that can be opened.
 
-    This is made to be used in conjuction with fsspec or s3fs. For example::
+    Giving the filenames to :class:`Scene` with valid transfer protocols will automatically
+    use this class so manual usage of this class is needed mainly for fine-grained control.
+
+    This class is made to be used in conjuction with fsspec or s3fs. For example::
 
         from satpy import Scene
 

satpy/scene.py

@@ -34,6 +34,7 @@
 from satpy.node import CompositorNode, MissingDependencies, ReaderNode
 from satpy.readers import load_readers
 from satpy.resample import get_area_def, prepare_resampler, resample_dataset
+from satpy.utils import convert_remote_files_to_fsspec, get_storage_options_from_reader_kwargs
 from satpy.writers import load_writer
 
 LOG = logging.getLogger(__name__)
@@ -106,21 +107,31 @@ def __init__(self, filenames=None, reader=None, filter_parameters=None,
                 sub-dictionaries to pass different arguments to different
                 reader instances.
 
+                Keyword arguments for remote file access are also given in this dictionary.
+                See `documentation <https://satpy.readthedocs.io/en/stable/remote_reading.html>`_
+                for usage examples.
+
         """
         self.attrs = dict()
+
+        storage_options, cleaned_reader_kwargs = get_storage_options_from_reader_kwargs(reader_kwargs)
+
         if filter_parameters:
-            if reader_kwargs is None:
-                reader_kwargs = {}
+            if cleaned_reader_kwargs is None:
+                cleaned_reader_kwargs = {}
             else:
-                reader_kwargs = reader_kwargs.copy()
-            reader_kwargs.setdefault('filter_parameters', {}).update(filter_parameters)
+                cleaned_reader_kwargs = cleaned_reader_kwargs.copy()
+            cleaned_reader_kwargs.setdefault('filter_parameters', {}).update(filter_parameters)
 
         if filenames and isinstance(filenames, str):
             raise ValueError("'filenames' must be a list of files: Scene(filenames=[filename])")
 
+        if filenames:
+            filenames = convert_remote_files_to_fsspec(filenames, storage_options)
+
         self._readers = self._create_reader_instances(filenames=filenames,
                                                       reader=reader,
-                                                      reader_kwargs=reader_kwargs)
+                                                      reader_kwargs=cleaned_reader_kwargs)
         self._datasets = DatasetDict()
         self._wishlist = set()
         self._dependency_tree = DependencyTree(self._readers)

satpy/tests/test_scene.py

@@ -632,6 +632,80 @@ def test_available_dataset_names_no_readers(self):
         name_list = scene.available_dataset_names(composites=True)
         assert name_list == []
 
+    def test_storage_options_from_reader_kwargs_no_options(self):
+        """Test getting storage options from reader kwargs.
+
+        Case where there are no options given.
+        """
+        filenames = ["s3://data-bucket/file1", "s3://data-bucket/file2", "s3://data-bucket/file3"]
+        with mock.patch('satpy.scene.load_readers'):
+            with mock.patch('fsspec.open_files') as open_files:
+                Scene(filenames=filenames)
+                open_files.assert_called_once_with(filenames)
+
+    def test_storage_options_from_reader_kwargs_single_dict_no_options(self):
+        """Test getting storage options from reader kwargs for remote files.
+
+        Case where a single dict is given for all readers without storage options.
+        """
+        filenames = ["s3://data-bucket/file1", "s3://data-bucket/file2", "s3://data-bucket/file3"]
+        reader_kwargs = {'reader_opt': 'foo'}
+        with mock.patch('satpy.scene.load_readers'):
+            with mock.patch('fsspec.open_files') as open_files:
+                Scene(filenames=filenames, reader_kwargs=reader_kwargs)
+                open_files.assert_called_once_with(filenames)
+
+    def test_storage_options_from_reader_kwargs_single_dict(self):
+        """Test getting storage options from reader kwargs.
+
+        Case where a single dict is given for all readers with some common storage options.
+        """
+        filenames = ["s3://data-bucket/file1", "s3://data-bucket/file2", "s3://data-bucket/file3"]
+        reader_kwargs = {'reader_opt': 'foo'}
+        expected_reader_kwargs = reader_kwargs.copy()
+        storage_options = {'option1': '1'}
+        reader_kwargs['storage_options'] = storage_options
+        with mock.patch('satpy.scene.load_readers') as load_readers:
+            with mock.patch('fsspec.open_files') as open_files:
+                Scene(filenames=filenames, reader_kwargs=reader_kwargs)
+                call_ = load_readers.mock_calls[0]
+                assert call_.kwargs['reader_kwargs'] == expected_reader_kwargs
+                open_files.assert_called_once_with(filenames, **storage_options)
+
+    def test_storage_options_from_reader_kwargs_per_reader(self):
+        """Test getting storage options from reader kwargs.
+
+        Case where each reader have their own storage options.
+        """
+        from copy import deepcopy
+
+        filenames = {
+            "reader1": ["s3://data-bucket/file1"],
+            "reader2": ["s3://data-bucket/file2"],
+            "reader3": ["s3://data-bucket/file3"],
+        }
+        storage_options_1 = {'option1': '1'}
+        storage_options_2 = {'option2': '2'}
+        storage_options_3 = {'option3': '3'}
+        reader_kwargs = {
+            "reader1": {'reader_opt_1': 'foo'},
+            "reader2": {'reader_opt_2': 'bar'},
+            "reader3": {'reader_opt_3': 'baz'},
+        }
+        expected_reader_kwargs = deepcopy(reader_kwargs)
+        reader_kwargs['reader1']['storage_options'] = storage_options_1
+        reader_kwargs['reader2']['storage_options'] = storage_options_2
+        reader_kwargs['reader3']['storage_options'] = storage_options_3
+
+        with mock.patch('satpy.scene.load_readers') as load_readers:
+            with mock.patch('fsspec.open_files') as open_files:
+                Scene(filenames=filenames, reader_kwargs=reader_kwargs)
+                call_ = load_readers.mock_calls[0]
+                assert call_.kwargs['reader_kwargs'] == expected_reader_kwargs
+                assert mock.call(filenames["reader1"], **storage_options_1) in open_files.mock_calls
+                assert mock.call(filenames["reader2"], **storage_options_2) in open_files.mock_calls
+                assert mock.call(filenames["reader3"], **storage_options_3) in open_files.mock_calls
+
 
 class TestFinestCoarsestArea:
     """Test the Scene logic for finding the finest and coarsest area."""