Merge pull request #299 from AstarVienna/dev_master

Release 0.7.1

README.md

@@ -2,9 +2,13 @@
 ## A telescope observation simulator for Python
 
 [![Build Status](https://github.com/AstarVienna/ScopeSim/actions/workflows/tests.yml/badge.svg)](https://github.com/AstarVienna/ScopeSim/actions/workflows/tests.yml/badge.svg)
-[![Documentation Status](https://readthedocs.org/projects/scopesim/badge/?version=latest)](https://scopesim.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/AstarVienna/ScopeSim/actions/workflows/minimumdependencies.yml/badge.svg)](https://github.com/AstarVienna/ScopeSim/actions/workflows/minimumdependencies.yml/badge.svg)
+[![Build Status](https://github.com/AstarVienna/ScopeSim/actions/workflows/notebooks_with_irdb_download.yml/badge.svg)](https://github.com/AstarVienna/ScopeSim/actions/workflows/notebooks_with_irdb_download.yml/badge.svg)
 
-[![Build Status](http://github-actions.40ants.com/AstarVienna/ScopeSim/matrix.svg)](https://github.com/AstarVienna/ScopeSim)
+[![Documentation Status](https://readthedocs.org/projects/scopesim/badge/?version=latest)](https://scopesim.readthedocs.io/en/latest)
+[![codecov](https://codecov.io/gh/AstarVienna/ScopeSim/graph/badge.svg)](https://codecov.io/gh/AstarVienna/ScopeSim)
+[![PyPI - Version](https://img.shields.io/pypi/v/ScopeSim)](https://pypi.org/project/ScopeSim/)
+[![Python Version Support](https://github-actions.40ants.com/AstarVienna/DevOps/matrix.svg?only=Tests.build.ubuntu-latest)](https://github.com/AstarVienna/ScopeSim)
 
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ScopeSim"
-version = "0.7.0"
+version = "0.7.1"
 description = "Generalised telescope observation simulator"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -20,10 +20,10 @@ classifiers=[
     "Topic :: Scientific/Engineering :: Astronomy",
 ]
 dependencies = [
-    "numpy>=1.19.5",
+    "numpy>=1.20.0",
     "scipy>=1.10.0",
     "astropy>=5.0",
-    "matplotlib>=3.2.0",
+    "matplotlib>=3.7.2",
     "pooch>=1.7.0",  # for scipy.datasets
 
     "docutils>=0.15",
@@ -36,7 +36,7 @@ dependencies = [
     "requests-cache>1.0",
 
     "synphot>=1.1.0",
-    "skycalc_ipy>=0.1.5",
+    "skycalc_ipy>=0.2.0",
     "anisocado>=0.3.0",
 ]
 
@@ -95,4 +95,6 @@ filterwarnings = [
     "ignore:Blowfish*:UserWarning",
     # Not sure what that is but it's everywhere...
     "ignore:'cgi'*:DeprecationWarning",
+    "ignore:The py23*:DeprecationWarning",
+    "ignore:datetime.datetime.utcfromtimestamp()*:DeprecationWarning",
 ]

scopesim/commands/user_commands.py

@@ -8,7 +8,7 @@
 import requests
 
 from .. import rc
-from ..utils import find_file
+from ..utils import find_file, top_level_catch
 
 __all__ = ["UserCommands"]
 
@@ -141,6 +141,7 @@ class UserCommands:
 
     """
 
+    @top_level_catch
     def __init__(self, **kwargs):
 
         self.cmds = copy.deepcopy(rc.__config__)
@@ -363,12 +364,7 @@ def add_packages_to_rc_search(local_path, package_list):
             # raise ValueError("Package could not be found: {}".format(pkg_dir))
             logging.warning("Package could not be found: %s", pkg_dir)
 
-        if pkg_dir in rc.__search_path__:
-            # if package is already in search_path, move it to the first place
-            ii = np.where(np.array(rc.__search_path__) == pkg_dir)[0][0]
-            rc.__search_path__.pop(ii)
-
-        rc.__search_path__ = [pkg_dir] + rc.__search_path__
+        rc.__search_path__.append_first(pkg_dir)
 
 
 def load_yaml_dicts(filename):

scopesim/effects/ter_curves_utils.py

@@ -1,6 +1,7 @@
 """TBA."""
 
 from pathlib import Path
+import logging
 
 import numpy as np
 from astropy import units as u
@@ -94,9 +95,15 @@ def download_svo_filter(filter_name, return_style="synphot"):
         silent=True,
     )
     if not path:
+        logging.debug("File not found in %s, downloading...", PATH_SVO_DATA)
         path = download_file(url, cache=True)
 
-    tbl = Table.read(path, format='votable')
+    try:
+        tbl = Table.read(path, format='votable')
+    except ValueError:
+        logging.error("Unable to load %s from %s.", filter_name, path)
+        raise
+
     wave = u.Quantity(tbl['Wavelength'].data.data, u.Angstrom, copy=False)
     trans = tbl['Transmission'].data.data
 

scopesim/optics/optical_element.py

@@ -65,7 +65,7 @@ def __init__(self, yaml_dict=None, **kwargs):
             self.meta.update({key: yaml_dict[key] for key in yaml_dict
                               if key not in {"properties", "effects"}})
             if "properties" in yaml_dict:
-                self.properties = yaml_dict["properties"]
+                self.properties = yaml_dict["properties"] or {}
             if "name" in yaml_dict:
                 self.properties["element_name"] = yaml_dict["name"]
             if "effects" in yaml_dict and len(yaml_dict["effects"]) > 0:

scopesim/optics/optical_train.py

@@ -16,7 +16,7 @@
 from ..commands.user_commands import UserCommands
 from ..detector import DetectorArray
 from ..effects import ExtraFitsKeywords
-from ..utils import from_currsys
+from ..utils import from_currsys, top_level_catch
 from ..version import version
 from .. import rc
 
@@ -76,6 +76,7 @@ class OpticalTrain:
 
     """
 
+    @top_level_catch
     def __init__(self, cmds=None):
         self.cmds = cmds
         self._description = self.__repr__()
@@ -107,6 +108,14 @@ def load(self, user_commands):
                              f"but is {type(user_commands)}")
 
         self.cmds = user_commands
+        # FIXME: Setting rc.__currsys__ to user_commands causes many problems:
+        #        UserCommands used SystemDict internally, but is itself not an
+        #        instance or subclas thereof. So rc.__currsys__ actually
+        #        changes type as a result of this line. On one hand, some other
+        #        code relies on this change, i.e. uses attributes from
+        #        UserCommands via rc.__currsys__, but on the other hand some
+        #        tests (now with proper patching) fail because of this type
+        #        change. THIS IS A PROBLEM!
         rc.__currsys__ = user_commands
         self.yaml_dicts = rc.__currsys__.yaml_dicts
         self.optics_manager = OpticsManager(self.yaml_dicts)
@@ -131,6 +140,7 @@ def update(self, **kwargs):
         self.detector_arrays = [DetectorArray(det_list, **kwargs)
                                 for det_list in opt_man.detector_setup_effects]
 
+    @top_level_catch
     def observe(self, orig_source, update=True, **kwargs):
         """
         Main controlling method for observing ``Source`` objects.
@@ -273,6 +283,7 @@ def prepare_source(self, source):
 
         return source
 
+    @top_level_catch
     def readout(self, filename=None, **kwargs):
         """
         Produce detector readouts for the observed image.

scopesim/rc.py

@@ -1,24 +1,32 @@
+# -*- coding: utf-8 -*-
+"""Global configurations for ScopeSim."""
+
 from pathlib import Path
 import yaml
 
-from .system_dict import SystemDict
+from copy import deepcopy
+
+from .system_dict import SystemDict, UniqueList
 
 __pkg_dir__ = Path(__file__).parent
 
-with open(__pkg_dir__/"defaults.yaml") as f:
-    dicts = list(yaml.full_load_all(f))
+with (__pkg_dir__ / "defaults.yaml").open(encoding="utf-8") as file:
+    dicts = list(yaml.full_load_all(file))
 
-user_rc_path = Path("~/.scopesim_rc.yaml").expanduser()
-if user_rc_path.exists():
-    with open(user_rc_path) as f:
-        dicts.extend(list(yaml.full_load_all(f)))
 
-__config__ = SystemDict(dicts)
-__currsys__ = __config__
+try:
+    with (Path.home() / ".scopesim_rc.yaml").open(encoding="utf-8") as file:
+        dicts.extend(list(yaml.full_load_all(file)))
+except FileNotFoundError:
+    pass
 
-__search_path__ = [__config__["!SIM.file.local_packages_path"],
-                   __pkg_dir__] + __config__["!SIM.file.search_path"]
 
-# if os.environ.get("READTHEDOCS") == "True" or "F:" in os.getcwd():
-#     extra_paths = ["../", "../../", "../../../", "../../../../"]
-#     __search_path__ = extra_paths + __search_path__
+__config__ = SystemDict(dicts)
+__currsys__ = deepcopy(__config__)
+
+# Order matters!
+__search_path__ = UniqueList([
+    Path(__config__["!SIM.file.local_packages_path"]).absolute(),
+    Path(__pkg_dir__).absolute(),
+    *[Path(pth).absolute() for pth in __config__["!SIM.file.search_path"]],
+])

scopesim/system_dict.py

@@ -3,7 +3,7 @@
 import logging
 from typing import TextIO
 from io import StringIO
-from collections.abc import Iterable, Mapping, MutableMapping
+from collections.abc import Iterable, MutableSequence, Mapping, MutableMapping
 
 from more_itertools import ilen
 
@@ -69,8 +69,19 @@ def __setitem__(self, key, value):
             self.dic[key] = value
 
     def __delitem__(self, key):
-        raise NotImplementedError("item deletion is not yet implemented for "
-                                  f"{self.__class__.__name__}")
+        # TODO: check this implementation with unit tests
+        if isinstance(key, str) and key.startswith("!"):
+            # TODO: these should be replaced with item.removeprefix("!")
+            #       once we can finally drop support for Python 3.8 UwU
+            *key_chunks, final_key = key[1:].split(".")
+            entry = self.dic
+            for key in key_chunks:
+                if not isinstance(entry, Mapping):
+                    raise KeyError(key)
+                entry = entry[key]
+            del entry[final_key]
+        else:
+            del self.dic[key]
 
     def _yield_subkeys(self, key, value):
         for subkey, subvalue in value.items():
@@ -116,6 +127,52 @@ def __str__(self) -> str:
         return output
 
 
+class UniqueList(MutableSequence):
+    """Ordered collection with unique elements."""
+
+    def __init__(self, initial: Iterable = None):
+        self._set = set()  # For uniqueness
+        self._list = []    # For order
+
+        if initial is not None:
+            self.extend(initial)
+
+    def __getitem__(self, index: int):
+        return self._list[index]
+
+    def __setitem__(self, index: int, value) -> None:
+        raise AttributeError(
+            f"{self.__class__.__name__} does not support item mutation, only "
+            "insertion, removal and reordering.")
+
+    def __delitem__(self, index: int) -> None:
+        self._set.discard(self._list.pop(index))
+
+    def __len__(self) -> int:
+        return len(self._set)
+
+    def __contains__(self, value) -> bool:
+        return value in self._set
+
+    def insert(self, index: int, value) -> None:
+        if value not in self:
+            self._set.add(value)
+            self._list.insert(index, value)
+
+    def append_first(self, value) -> None:
+        """
+        Append element to the front of the list.
+
+        If the element is already present in the list, move it to the front.
+        """
+        if value in self:
+            self.remove(value)
+        self.insert(0, value)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self._list!r})"
+
+
 def recursive_update(old_dict: MutableMapping, new_dict: Mapping) -> MutableMapping:
     if new_dict is not None:
         for key in new_dict:

scopesim/tests/conftest.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""Global fixtures for pytest."""
+
+from pathlib import Path
+
+import pytest
+from unittest.mock import patch
+
+import scopesim as sim
+from scopesim.system_dict import UniqueList
+
+
+MOCK_DIR = Path(__file__).parent / "mocks"
+
+sim.rc.__currsys__["!SIM.file.error_on_missing_file"] = True
+
+
+@pytest.fixture(scope="package")
+def mock_dir():
+    """Path to mock directory."""
+    return MOCK_DIR
+
+
+@pytest.fixture(scope="package")
+def mock_path():
+    """Path to mock files."""
+    return MOCK_DIR / "files"
+
+
+@pytest.fixture(scope="class")
+def patch_mock_path(mock_path):
+    """Patch __search_path__ with test files mock path.
+
+    Use only when needed internally, refer to filenames in tests using full
+    absolute path (with the help of the mock_path fixture).
+    """
+    with patch("scopesim.rc.__search_path__", [mock_path]):
+        yield
+
+
+@pytest.fixture(scope="class")
+def patch_all_mock_paths(mock_dir):
+    with patch("scopesim.rc.__search_path__", UniqueList([mock_dir])):
+        patched = {"!SIM.file.local_packages_path": str(mock_dir)}
+        with patch.dict("scopesim.rc.__config__", patched):
+            with patch.dict("scopesim.rc.__currsys__", patched):
+                yield
+
+
+@pytest.fixture(scope="package")
+def mock_path_yamls():
+    """Path to mock yaml files."""
+    return MOCK_DIR / "yamls"
+
+
+@pytest.fixture(scope="package")
+def mock_path_micado():
+    """Path to MICADO mock files."""
+    return MOCK_DIR / "MICADO_SCAO_WIDE"
+
+
+@pytest.fixture(scope="class")
+def patch_mock_path_micado(mock_path_micado):
+    """Patch __search_path__ with MICADO mock path.
+
+    Use only when needed internally, refer to filenames in tests using full
+    absolute path (with the help of the mock_path_micado fixture).
+    """
+    with patch("scopesim.rc.__search_path__", [mock_path_micado]):
+        yield
+
+
+@pytest.fixture(scope="function")
+def no_file_error():
+    """Patch currsys to avoid missing file error."""
+    patched = {"!SIM.file.error_on_missing_file": False}
+    with patch.dict("scopesim.rc.__currsys__", patched):
+        yield
+
+
+@pytest.fixture(scope="function")
+def protect_currsys():
+    """Prevent modification of global currsys."""
+    with patch("scopesim.rc.__currsys__"):
+        yield