Merge branch 'master' into bugfix-export-imageiov3

.github/workflows/ci.yaml

@@ -117,7 +117,7 @@ jobs:
         if: matrix.experimental == true
         run: |
           python -m pip install \
-          --index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple/ \
+          --index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \
           --trusted-host pypi.anaconda.org \
           --no-deps --pre --upgrade \
           matplotlib \
@@ -145,8 +145,7 @@ jobs:
           pytest --cov=uwsift uwsift/tests
 
       - name: Coveralls Parallel
-        # See https://github.com/AndreMiras/coveralls-python-action/pull/16
-        uses: djhoese/coveralls-python-action@feature-cython-coverage
+        uses: AndreMiras/coveralls-python-action@develop
         with:
           flag-name: run-${{ matrix.test_number }}
           parallel: true
@@ -157,7 +156,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Coveralls Finished
-        # See https://github.com/AndreMiras/coveralls-python-action/pull/16
-        uses: djhoese/coveralls-python-action@feature-cython-coverage
+        uses: AndreMiras/coveralls-python-action@develop
         with:
           parallel-finished: true

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: '^$'
 fail_fast: false
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
         language_version: python3
@@ -14,7 +14,7 @@ repos:
       - id: isort
         language_version: python3
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         language_version: python3
@@ -35,7 +35,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.3.0'  # Use the sha / tag you want to point at
+    rev: 'v1.5.1'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

doc/source/configuration/external_satpy.rst

@@ -1,54 +1,37 @@
 Configuring External Satpy Components
--------------------------------------
+=====================================
 
-Replacing Satpy by External Installation
-========================================
+SIFT can use external :ref:`Satpy component configuration <satpy:component_configuration>` folder,
+that hosts extra ``readers``, ``composites``, ``enhancements`` and ``areas`` definitions.
+To use the external satpy component configuration it is necessary to define either
+``satpy_extra_config_path`` in the personal user configs (e.g. inside a file called `external_satpy.yaml`)::
 
-SIFT can be instructed to import Satpy modules from another location than
-from the site packages of the active Python environment when the following
-setting points to an appropriate package directory::
+    satpy_extra_config_path: [directory path]
 
-   satpy_import_path: [directory path]
+or the environment variable ``SATPY_CONFIG_PATH`` as described `here <https://satpy.readthedocs.io/en/stable/config.html#config-path-setting>`_.
 
-For example you can use your development version of Satpy cloned directly from
-GitHub to ``/home/me/development/satpy`` by configuring::
-
-   satpy_import_path: "/home/me/development/satpy/satpy"
-
-or setting the according environment variable before starting SIFT::
-
-   export UWSIFT_SATPY_IMPORT_PATH="/home/me/development/satpy/satpy"
-
-It is your responsibility to make sure the setting points to a suitable Satpy
-package: If the given path doesn't point to a Python package directory or not to
-one providing Satpy, the application may exit immediately throwing Exceptions.
-
-Using Extra Readers
-===================
+Example of external readers configuration
+-----------------------------------------
 
 Several data formats which are or will be produced by EUMETSAT need special
 readers which are not (yet) part of the official Satpy distribution. EUMETSAT
 maintains a Git repository ``satpy/local_readers`` on their `GitLab
 <https://gitlab.eumetsat.int/satpy/local_readers>`_ providing these special
-readers. To use these readers in addition to those included in Satpy the path to
-the root directory of a clone of this repository must be configured via the
-following setting first::
-
-    satpy_extra_readers_import_path: [directory path]
+readers. To use these readers it is neccsary to put them into folder: ``satpy_extra_config_path/readers``.
 
 Furthermore the desired readers need to be added to the configuration
 ``data_reading.readers`` and their reader specific configuration as well (see
 **TODO**).
 
 For example assuming that the repository has been cloned as follows::
 
-    git clone https://gitlab.eumetsat.int/satpy/local_readers.git /path/to/clone/of/satpy/local_readers
+    git clone https://gitlab.eumetsat.int/satpy/local_readers.git /path/to/satpy_extra_config_path/readers
 
 the readers for the *FCI L1 Landmark Locations Catalogue*, *FCI L1 GEOOBS
 Landmarks* (landmark locations) and *FCI L1 GEOOBS Landmark Matching Results*
 (landmark navigation error) can be made available in SIFT with::
 
-    satpy_extra_readers_import_path: /path/to/clone/of/satpy/local_readers
+    satpy_extra_config_path: /path/to/satpy_extra_config_path
 
     data_reading:
       readers:

doc/source/configuration/readers.rst

@@ -6,7 +6,7 @@ General Configuration
 
 The readers offered by the Open File Wizard to be used for loading are a subset
 of the readers provided by Satpy and - if configured accordingly via
-``satpy_extra_readers_import_path`` - in a directory of additional readers.
+``satpy_extra_config_path`` - in a directory of additional readers.
 The list of these readers must be configured, e.g. as follows::
 
   data_reading:

setup.py

@@ -199,7 +199,7 @@ def run(self):
     include_package_data=True,
     install_requires=[
         "appdirs",
-        "donfig",
+        "donfig>=0.8.1",
         "h5py",
         "imageio",
         "av",

uwsift/__init__.py

@@ -6,6 +6,7 @@
 from importlib.machinery import PathFinder
 from importlib.util import spec_from_file_location
 
+import satpy
 from donfig import Config
 
 from uwsift.common import ImageDisplayMode
@@ -46,13 +47,14 @@ def init_user_config_dirs(user_config_dirs: list):
 init_user_config_dirs(USER_CONFIG_PATHS)
 
 print(f"Reading configuration from:\n\t{CONFIG_PATHS}", file=sys.stderr)
-config = Config("uwsift", paths=CONFIG_PATHS)
+deprecations = {"satpy_extra_readers_import_path": "satpy_extra_config_path"}
+config = Config("uwsift", paths=CONFIG_PATHS, deprecations=deprecations)
 
 
 def overwrite_import(package_name: str, custom_import_path: str, *, verbose=True):
     if (custom_import_path is not None) and (not os.path.exists(custom_import_path)):
         raise FileNotFoundError(
-            f"Package '{package_name}' " f"doesn't exist at given custom import path '{custom_import_path}'"
+            f"Package '{package_name}' doesn't exist at given custom import path '{custom_import_path}'"
         )
 
     class CustomPathFinder(PathFinder):
@@ -83,14 +85,11 @@ def find_spec(cls, fullname: str, path=None, target=None):
     sys.meta_path.insert(0, CustomPathFinder)
 
 
-satpy_import_path = config.get("satpy_import_path", None)
-if satpy_import_path is not None:
-    overwrite_import("satpy", satpy_import_path)
-
-satpy_extra_readers_import_path = config.get("satpy_extra_readers_import_path", None)
-if satpy_extra_readers_import_path is not None:
-    sys.path.insert(0, satpy_extra_readers_import_path)
-    os.environ["SATPY_CONFIG_PATH"] = satpy_extra_readers_import_path
+# Add additional satpy configuration paths
+satpy_config_path_yml = config.get("satpy_extra_config_path", None)
+if satpy_config_path_yml is not None:
+    satpy.config.set(config_path=satpy.config.get("config_path") + [satpy_config_path_yml])
+    sys.path.insert(0, satpy_config_path_yml)
 
 
 def _map_str_to_image_display_mode(image_display_mode_str: str) -> ImageDisplayMode:

uwsift/etc/SIFT/config/external_satpy_and_readers.yaml → uwsift/etc/SIFT/config/external_satpy.yaml

@@ -2,5 +2,5 @@
 # This can be used during development to load a different version of Satpy without the need to package it first.
 # satpy_import_path: "/path/to/your/satpy/repo/clone/satpy"
 
-# Define the path to a folder containing custom Satpy readers to be used in SIFT.
-# satpy_extra_readers_import_path: "path/to/your/local_readers/repo/clone/local_readers"
+# Define the path to a folder containing custom Satpy configuration of readers, composites, enhancements and areas to be used in SIFT.
+# satpy_extra_config_path: "path/to/your/local_component_configuration"

uwsift/tests/test_satpy_import.py

@@ -1,8 +1,6 @@
 import importlib
 import importlib.util
 import os
-import re
-import shutil
 import subprocess
 import sys
 from ast import literal_eval
@@ -21,36 +19,26 @@ def get_python_path() -> str:
 
 
 @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="can only set the config directory on Linux")
-def test_config_satpy_import_path(tmp_path):
-    satpy_module = importlib.import_module("satpy")
-
+def test_xdg_config_home(tmp_path):
     # get the folder structure of the config directory
     uwsift_module = importlib.import_module("uwsift")
     relative_config_dir = os.path.relpath(uwsift_module.USER_CONFIG_PATHS[0], appdirs.user_config_dir())
 
     # Don't import the `uwsift` module using the current Python Interpreter, because the code
     # in the __init__.py is just executed once. Use a subprocess, such that the code can be
     # executed multiple times with different config files and corrupted Satpy modules.
-    def check_uwsift_paths(
-        xdg_config_home: Union[str, None],
-        base_config_dir: str,
-        overwritten_satpy_import_path: Union[str, None],
-        satpy_init_path: str,
-    ):
+    def check_uwsift_paths(xdg_config_home: Union[str, None], base_config_dir: str) -> None:
         if xdg_config_home is None:
             modified_env = {}
         else:
             modified_env = {"XDG_CONFIG_HOME": xdg_config_home}
 
         # Use stderr, because other print statements will confuse the `literal_eval`.
-        # If an import is overwritten, then `overwrite_import` will output to stdout.
         command = [
             get_python_path(),
             "-c",
-            "import sys, uwsift, satpy, satpy.readers\n"
-            "print({'base_config_dir': uwsift.USER_CONFIG_PATHS[0], "
-            "'overwritten_satpy_import_path': uwsift.config.get('satpy_import_path', None), "
-            "'satpy_init_path': satpy.__file__})",
+            "import uwsift; import sys, satpy, satpy.readers\n"
+            "print({'base_config_dir': uwsift.USER_CONFIG_PATHS[0]})",
         ]
         working_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
         process = subprocess.run(
@@ -60,62 +48,12 @@ def check_uwsift_paths(
 
         # Can we configure the config file path with the environment variable `XDG_CONFIG_HOME`?
         assert results["base_config_dir"] == base_config_dir
-        # Is the Satpy module path correctly read from the config file `general_settings.yml`?
-        assert results["overwritten_satpy_import_path"] == overwritten_satpy_import_path
-        # Where is the `__init__.py` from Satpy located?
-        assert results["satpy_init_path"] == satpy_init_path
 
-    # If the environment variable `XDG_CONFIG_HOME` isn't set, then we search for the config in the user
-    # specific config directory. If the file exists, we search for the `satpy_import_path`. Otherwise the
-    # `satpy_import_path` must be None, in which case we don't overwrite the default Satpy import path.
-    user_general_settings_path = os.path.join(uwsift_module.USER_CONFIG_PATHS[0], "general_settings.yml")
-    if os.path.exists(user_general_settings_path):
-        if os.path.isfile(user_general_settings_path):
-            with open(user_general_settings_path) as file:
-                import_path_regex = re.compile('^satpy_import_path: "(.*)"$')
-                for line in file.readlines():
-                    match = import_path_regex.match(line)
-                    if match:
-                        user_satpy_import_path = match.group(1)
-                        break
-                else:
-                    # the file is empty or `satpy_import_path` is commented out
-                    user_satpy_import_path = None
-        else:
-            raise FileNotFoundError(f"general_settings.yml isn't a file: {user_general_settings_path}")
-    else:
-        user_satpy_import_path = None
-    check_uwsift_paths(None, uwsift_module.USER_CONFIG_PATHS[0], user_satpy_import_path, satpy_module.__file__)
+    check_uwsift_paths(None, uwsift_module.USER_CONFIG_PATHS[0])
 
+    # test that XDG_CONFIG_HOME changes which configuration directory is used
     # recreate that folder structure inside the temp directory `config`
     tmp_config_dir = os.path.join(tmp_path, "config")
     sift_tmp_config_dir = os.path.join(tmp_config_dir, relative_config_dir)
     os.makedirs(sift_tmp_config_dir)
-
-    # `satpy_import_path` is None, because the general_settings.yml doesn't exist yet.
-    # Therefore the Satpy import path won't be overwritten and the default path is used.
-    check_uwsift_paths(tmp_config_dir, sift_tmp_config_dir, None, satpy_module.__file__)
-
-    def set_satpy_import_path(import_path: str):
-        """create a config file, which points to the Satpy module in the temp `satpy` directory"""
-        general_settings_path = os.path.join(sift_tmp_config_dir, "general_settings.yml")
-        with open(general_settings_path, "w") as file:
-            file.write(f'satpy_import_path: "{tmp_satpy_dir}"\n')
-
-    tmp_satpy_dir = os.path.join(tmp_path, "satpy")
-    set_satpy_import_path(tmp_satpy_dir)
-    with pytest.raises((FileNotFoundError, subprocess.CalledProcessError)):
-        # The temp `Satpy` directory is still empty, thus a FileNotFoundError will be raised.
-        # Note: Only the first parameter is used, because the exception will be raised before the asserts.
-        check_uwsift_paths(tmp_config_dir, sift_tmp_config_dir, None, tmp_satpy_dir)
-
-    # copy the Satpy module to the temp directory `satpy`
-    shutil.copytree(os.path.dirname(satpy_module.__file__), tmp_satpy_dir)
-    tmp_satpy_init = os.path.join(tmp_satpy_dir, "__init__.py")
-    check_uwsift_paths(tmp_config_dir, sift_tmp_config_dir, tmp_satpy_dir, tmp_satpy_init)
-
-    # delete the `readers` subpackage from the copied Satpy installation
-    readers_submodule = os.path.join(tmp_satpy_dir, "readers")
-    shutil.rmtree(readers_submodule)
-    with pytest.raises((ImportError, subprocess.CalledProcessError)):
-        check_uwsift_paths(tmp_config_dir, sift_tmp_config_dir, tmp_satpy_dir, tmp_satpy_init)
+    check_uwsift_paths(tmp_config_dir, sift_tmp_config_dir)