From b751649a5427929987e9734bcc3e38376bc793fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Tue, 23 Aug 2022 09:06:23 +0200 Subject: [PATCH 1/6] refactor: add yaml loader as argument to available_readers --- doc/source/reader_table.py | 7 ++++++- satpy/readers/__init__.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/source/reader_table.py b/doc/source/reader_table.py index 3982728f47..a192c28e7a 100644 --- a/doc/source/reader_table.py +++ b/doc/source/reader_table.py @@ -20,6 +20,11 @@ from satpy.readers import available_readers +try: + from yaml import BaseLoader +except ImportError: + from yaml import BaseLoader # type: ignore + def rst_table_row(columns=None): """Create one row for a rst table. @@ -72,7 +77,7 @@ def generate_reader_table(): table = [rst_table_header("Satpy Readers", header=["Description", "Reader name", "Status", "fsspec support"], widths=[45, 25, 30, 30])] - reader_configs = available_readers(as_dict=True) + reader_configs = available_readers(as_dict=True, yaml_loader=BaseLoader) for rc in reader_configs: table.append(rst_table_row([rc.get("long_name", "").rstrip("\n"), rc.get("name", ""), rc.get("status", ""), rc.get("supports_fsspec", "false")])) diff --git a/satpy/readers/__init__.py b/satpy/readers/__init__.py index a789623f62..2094976d25 100644 --- a/satpy/readers/__init__.py +++ b/satpy/readers/__init__.py @@ -370,7 +370,7 @@ def get_valid_reader_names(reader): return new_readers -def available_readers(as_dict=False): +def available_readers(as_dict=False, yaml_loader=UnsafeLoader): """Available readers based on current configuration. Args: @@ -385,7 +385,7 @@ def available_readers(as_dict=False): readers = [] for reader_configs in configs_for_reader(): try: - reader_info = read_reader_config(reader_configs) + reader_info = read_reader_config(reader_configs, loader=yaml_loader) except (KeyError, IOError, yaml.YAMLError): LOG.debug("Could not import reader config from: %s", reader_configs) LOG.debug("Error loading YAML", exc_info=True) From ec971f975a30f1a01c4e6b188f459f45dc56040b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Tue, 23 Aug 2022 13:54:08 +0200 Subject: [PATCH 2/6] refactor: move table summary to top of table --- doc/source/_static/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/_static/main.js b/doc/source/_static/main.js index 6fcd3ce79a..188a335e71 100644 --- a/doc/source/_static/main.js +++ b/doc/source/_static/main.js @@ -1,5 +1,6 @@ $(document).ready( function () { $('table.datatable').DataTable( { - "paging": false + "paging": false, + "dom": 'lfitp' } ); } ); From 80f2a1dba5ca9f66599b5df49578528a7f4473a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Tue, 23 Aug 2022 15:27:31 +0200 Subject: [PATCH 3/6] refactor: bump required pyyaml version --- doc/source/reader_table.py | 7 ++----- satpy/composites/config_loader.py | 6 +----- satpy/plugin_base.py | 6 +----- satpy/readers/__init__.py | 6 +----- satpy/readers/yaml_reader.py | 10 ++-------- satpy/utils.py | 7 +------ satpy/writers/__init__.py | 7 +------ setup.py | 2 +- utils/convert_to_ninjotiff.py | 7 +------ 9 files changed, 11 insertions(+), 47 deletions(-) diff --git a/doc/source/reader_table.py b/doc/source/reader_table.py index a192c28e7a..1c6760a390 100644 --- a/doc/source/reader_table.py +++ b/doc/source/reader_table.py @@ -18,12 +18,9 @@ # along with satpy. If not, see . """Module for autogenerating reader table from config files.""" -from satpy.readers import available_readers +from yaml import BaseLoader -try: - from yaml import BaseLoader -except ImportError: - from yaml import BaseLoader # type: ignore +from satpy.readers import available_readers def rst_table_row(columns=None): diff --git a/satpy/composites/config_loader.py b/satpy/composites/config_loader.py index 58f6b2a2bb..4550ca58b9 100644 --- a/satpy/composites/config_loader.py +++ b/satpy/composites/config_loader.py @@ -25,11 +25,7 @@ from typing import Callable, Iterable import yaml - -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore +from yaml import UnsafeLoader import satpy from satpy import DataID, DataQuery diff --git a/satpy/plugin_base.py b/satpy/plugin_base.py index 65660ba1f0..ee19341796 100644 --- a/satpy/plugin_base.py +++ b/satpy/plugin_base.py @@ -20,11 +20,7 @@ import logging import yaml - -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore +from yaml import UnsafeLoader from satpy._config import config_search_paths from satpy.utils import recursive_dict_update diff --git a/satpy/readers/__init__.py b/satpy/readers/__init__.py index 2094976d25..449f48ab5c 100644 --- a/satpy/readers/__init__.py +++ b/satpy/readers/__init__.py @@ -26,11 +26,7 @@ from functools import total_ordering import yaml - -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore +from yaml import UnsafeLoader from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 2e3203e5a8..97eb7510bd 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -26,22 +26,16 @@ from collections import OrderedDict, deque from contextlib import suppress from fnmatch import fnmatch +from functools import cached_property from weakref import WeakValueDictionary import numpy as np import xarray as xr import yaml - -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore - -from functools import cached_property - from pyresample.boundary import AreaDefBoundary, Boundary from pyresample.geometry import AreaDefinition, StackedAreaDefinition, SwathDefinition from trollsift.parser import globify, parse +from yaml import UnsafeLoader from satpy import DatasetDict from satpy.aux_download import DataDownloadMixin diff --git a/satpy/utils.py b/satpy/utils.py index e1489a9c19..26efc90bf5 100644 --- a/satpy/utils.py +++ b/satpy/utils.py @@ -31,15 +31,10 @@ import numpy as np import xarray as xr import yaml -from yaml import BaseLoader +from yaml import BaseLoader, UnsafeLoader from satpy import CHUNK_SIZE -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore - _is_logging_on = False TRACE_LEVEL = 5 diff --git a/satpy/writers/__init__.py b/satpy/writers/__init__.py index f31a6008f6..655b42571c 100644 --- a/satpy/writers/__init__.py +++ b/satpy/writers/__init__.py @@ -29,14 +29,9 @@ import numpy as np import xarray as xr import yaml - -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore - from trollimage.xrimage import XRImage from trollsift import parser +from yaml import UnsafeLoader from satpy import CHUNK_SIZE from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config diff --git a/setup.py b/setup.py index 0e700c37ae..cdddac0058 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ pass requires = ['numpy >=1.13', 'pillow', 'pyresample >=1.24.0', 'trollsift', - 'trollimage >1.10.1', 'pykdtree', 'pyyaml', 'xarray >=0.10.1, !=0.13.0', + 'trollimage >1.10.1', 'pykdtree', 'pyyaml >=5.1', 'xarray >=0.10.1, !=0.13.0', 'dask[array] >=0.17.1', 'pyproj>=2.2', 'zarr', 'donfig', 'appdirs', 'pooch', 'pyorbital'] diff --git a/utils/convert_to_ninjotiff.py b/utils/convert_to_ninjotiff.py index 1c3eef595e..e457ee35e3 100644 --- a/utils/convert_to_ninjotiff.py +++ b/utils/convert_to_ninjotiff.py @@ -30,17 +30,12 @@ import os import yaml +from yaml import UnsafeLoader from satpy import Scene from satpy.pyresample import get_area_def from satpy.utils import debug_on -try: - from yaml import UnsafeLoader -except ImportError: - from yaml import Loader as UnsafeLoader # type: ignore - - debug_on() parser = argparse.ArgumentParser(description='Turn an image into a NinjoTiff.') From 86448f49a0f2b236d2e6a3c115fee754d1ee6eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Wed, 24 Aug 2022 16:39:11 +0200 Subject: [PATCH 4/6] test: add test for available_readers with BaseLoader --- satpy/tests/test_readers.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/satpy/tests/test_readers.py b/satpy/tests/test_readers.py index 6a30ba09fa..c11685b9b3 100644 --- a/satpy/tests/test_readers.py +++ b/satpy/tests/test_readers.py @@ -17,7 +17,9 @@ # satpy. If not, see . """Test classes and functions in the readers/__init__.py module.""" +import builtins import os +import sys import unittest from contextlib import suppress from unittest import mock @@ -55,6 +57,8 @@ }, } +real_import = builtins.__import__ + def make_dataid(**items): """Make a data id.""" @@ -625,6 +629,11 @@ def test_old_reader_name_mapping(self): class TestYAMLFiles(unittest.TestCase): """Test and analyze the reader configuration files.""" + def setUp(self): + """Set up monkeypatch.""" + from _pytest.monkeypatch import MonkeyPatch + self.monkeypatch = MonkeyPatch() + def test_filename_matches_reader_name(self): """Test that every reader filename matches the name in the YAML.""" import yaml @@ -662,6 +671,28 @@ def test_available_readers(self): self.assertIn('name', reader_info) self.assertEqual(reader_infos, sorted(reader_infos, key=lambda reader_info: reader_info['name'])) + def test_available_readers_base_loader(self): + """Test the 'available_readers' function for yaml loader type BaseLoader.""" + import yaml + + from satpy import available_readers + from satpy._config import glob_config + + def patched_import_error(name, globals=None, locals=None, fromlist=(), level=0): + if name in ('netcdf4', ): + raise ImportError(f"Mocked import error {name}") + return real_import(name, globals=globals, locals=locals, fromlist=fromlist, level=level) + + self.monkeypatch.delitem(sys.modules, 'netcdf4', raising=False) + self.monkeypatch.setattr(builtins, '__import__', patched_import_error) + + with pytest.raises(ImportError): + import netcdf4 # noqa: F401 + + reader_names = available_readers(yaml_loader=yaml.BaseLoader) + self.assertIn('abi_l1b', reader_names) # needs netcdf4 + self.assertEqual(len(reader_names), len(list(glob_config('readers/*.yaml')))) + class TestGroupFiles(unittest.TestCase): """Test the 'group_files' utility function.""" From 361263cc6d80577d6dffddd50c98099595c523bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Wed, 24 Aug 2022 16:59:21 +0200 Subject: [PATCH 5/6] refactor: add another reader name to check --- satpy/tests/test_readers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/tests/test_readers.py b/satpy/tests/test_readers.py index c11685b9b3..3f76e63596 100644 --- a/satpy/tests/test_readers.py +++ b/satpy/tests/test_readers.py @@ -691,6 +691,7 @@ def patched_import_error(name, globals=None, locals=None, fromlist=(), level=0): reader_names = available_readers(yaml_loader=yaml.BaseLoader) self.assertIn('abi_l1b', reader_names) # needs netcdf4 + self.assertIn('viirs_l1b', reader_names) self.assertEqual(len(reader_names), len(list(glob_config('readers/*.yaml')))) From 340602681f0507df80e7b1a8879060d4041fd6c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 25 Aug 2022 09:46:28 +0200 Subject: [PATCH 6/6] doc: add loader type argument to docstring --- satpy/readers/__init__.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/satpy/readers/__init__.py b/satpy/readers/__init__.py index 449f48ab5c..de9efd26e4 100644 --- a/satpy/readers/__init__.py +++ b/satpy/readers/__init__.py @@ -371,11 +371,13 @@ def available_readers(as_dict=False, yaml_loader=UnsafeLoader): Args: as_dict (bool): Optionally return reader information as a dictionary. - Default: False + Default: False. + yaml_loader (Optional[Union[yaml.BaseLoader, yaml.FullLoader, yaml.UnsafeLoader]]): + The yaml loader type. Default: ``yaml.UnsafeLoader``. - Returns: List of available reader names. If `as_dict` is `True` then - a list of dictionaries including additionally reader information - is returned. + Returns: + Union[list[str], list[dict]]: List of available reader names. If `as_dict` is `True` then + a list of dictionaries including additionally reader information is returned. """ readers = [] @@ -450,12 +452,12 @@ def find_files_and_readers(start_time=None, end_time=None, base_dir=None, missing_ok (bool): If False (default), raise ValueError if no files are found. If True, return empty dictionary if no files are found. - fs (FileSystem): Optional, instance of implementation of - fsspec.spec.AbstractFileSystem (strictly speaking, - any object of a class implementing ``.glob`` is - enough). Defaults to searching the local filesystem. + fs (:class:`fsspec.spec.AbstractFileSystem`): Optional, instance of implementation of + :class:`fsspec.spec.AbstractFileSystem` (strictly speaking, any object of a class implementing + ``.glob`` is enough). Defaults to searching the local filesystem. - Returns: Dictionary mapping reader name string to list of filenames + Returns: + dict: Dictionary mapping reader name string to list of filenames """ reader_files = {}