Merge branch 'run_kwargs_fix' of github.com-rueberger:rueberger/sacre…

…d into run_kwargs_fix

* 'run_kwargs_fix' of github.com-rueberger:rueberger/sacred:
  Release 0.7.5
  update dependencies to safer version of SQLAlchemy
  fixed numpy deprecation warning
  fix yaml deprecation warning
  Make config read only (IDSIA#472)
  Remove broken codacy badge (IDSIA#486)
  Add queue based observer (IDSIA#471)
  Run CI against python 36 and 37 (IDSIA#485)
  Make failed mongo observer dump configurable (IDSIA#462)
  Make stale time longer (IDSIA#484)
  Fix a bug where a unicode char in README.rst would fail install (IDSIA#481)

.github/stale.yml

@@ -1,5 +1,5 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 30
+daysUntilStale: 90
 
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7

.travis.yml

@@ -1,5 +1,6 @@
 language: python
 sudo: false
+dist: xenial
 install:
   - pip install tox
 matrix:
@@ -8,13 +9,17 @@ matrix:
     env: TOX_ENV=py27
   - python: "3.5"
     env: TOX_ENV=py35
+  - python: "3.6"
+    env: TOX_ENV=py36
+  - python: "3.7"
+    env: TOX_ENV=py37
   - python: "3.6"
     env: TOX_ENV=tensorflow-1
   - python: "3.6"
     env: TOX_ENV=tensorflow-2
   - python: "3.6"
     env: TOX_ENV=flake8
-  - python: "3.5"
+  - python: "3.6"
     env: TOX_ENV=coverage
 script:
   - tox -e $TOX_ENV

HISTORY.rst

@@ -1,6 +1,34 @@
 Release History
 ---------------
 
+0.7.5 (2019-06-20)
+++++++++++++++++++
+Last release to support Python 2.7.
+
+* Feature: major improvements to error reporting (thanks @thequilo)
+* Feature: added print_named_configs command
+* Feature: added option to add metadata to artifacts (thanks @jarnoRFB)
+* Feature: content type detection for artifacts (thanks @jarnoRFB)
+* Feature: automatic seeding for pytorch (thanks @srossi93)
+* Feature: add proxy support to telegram observer (thanks @brickerino)
+* Feature: made MongoObserver fail dump dir configurable (thanks @jarnoRFB)
+* Feature: added queue-based observer that better handles unreliable connections (thanks @jarnoRFB)
+* Bugfix: some fixes to stdout capturing
+* Bugfix: FileStorageObserver now creates directories only when starting a run (#329; thanks @thomasjpfan)
+* Bugfix: Fixed config_hooks (#326; thanks @thomasjpfan)
+* Bugfix: Fixed a crash when overwriting non-dict config entries with dicts (#325; thanks @thomasjpfan)
+* Bugfix: fixed problem with running in conda environment (#341)
+* Bugfix: numpy aware config change detection (#344)
+* Bugfix: allow dependencies to be compiled libraries (thanks @jnphilipp)
+* Bugfix: output colorization now works on 256 and 16 color terminals (thanks @bosr)
+* Bugfix: fixed problem with tinydb observer logging (#327; thanks @michalgregor)
+* Bugfix: ignore folders that have the same name as a named_config (thanks @boeddeker)
+* Bugfix: setup no longer overwrites pre-configured root logger (thanks @thequilo)
+* Bugfix: compatibility with tensorflow 2.0 (thanks @tarik, @gabrieldemarmiesse)
+* Bugfix: fixed exception when no tee is available for stdout capturing (thanks @greg-farquhar)
+* Bugfix: fixed concurrency issue with FileStorageObserver (thanks @dekuenstle)
+
+
 0.7.4 (2018-06-12)
 ++++++++++++++++++
 * Bugfix: fixed problem with postgres backend of SQLObserver (thanks @bensternlieb)

README.rst

@@ -8,7 +8,7 @@ Sacred
 
 |pypi| |py_versions| |license| |rtfd| |doi|
 
-|unix_build| |windows_build| |coverage| |code_quality| |codacy|
+|unix_build| |windows_build| |coverage| |code_quality|
 
 
 
@@ -250,8 +250,5 @@ in Proceedings of the 15th Python in Science Conference (SciPy 2017), Austin, Te
     :target: https://scrutinizer-ci.com/g/IDSIA/sacred/
     :alt: Code Scrutinizer Quality
 
-.. |codacy| image:: https://img.shields.io/codacy/acb7bba4467e47deaf260d6df5c0279f.svg?style=flat
-    :target: https://www.codacy.com/app/qwlouse/sacred
-    :alt: Codacity rating
 
 

dev-requirements.txt

@@ -22,7 +22,7 @@ PyYAML>=4.2b1
 scandir==1.4
 sentinels==1.0.0
 smmap2==2.0.1
-SQLAlchemy==1.1.4
+SQLAlchemy>=1.3.0
 tinydb==3.2.1
 tinydb-serialization==1.0.3
 wrapt==1.10.8

docs/configuration.rst

@@ -402,6 +402,14 @@ You will still get an appropriate error in the following cases:
     hide some missing value errors from you, by (unintentionally) filling them
     in from the configuration.
 
+.. note::
+    Configuration values should not be changed in a captured function
+    because those changes cannot be recorded by the sacred experiment and can
+    lead to confusing and unintended behaviour.
+    Sacred will raise an Exception if you try to write to a nested
+    configuration item. You can disable this (not recommended) by setting
+    ``SETTINGS.CONFIG.READ_ONLY_CONFIG = False``.
+
 .. _special_values:
 
 Special Values

docs/settings.rst

@@ -37,6 +37,10 @@ Here is a brief list of all currently available options.
   * ``IGNORED_COMMENTS`` *(default: ['^pylint:', '^noinspection'])*
     list of regex patterns to filter out certain IDE or linter directives
     from in-line comments in the documentation.
+  * ``READ_ONLY_CONFIG`` *(default: True)*
+    Make the configuration read-only inside of captured functions. This
+    only works to a limited extend because custom types cannot be
+    controlled.
 
 * ``HOST_INFO``
 

sacred/__about__.py

@@ -11,7 +11,7 @@
 
 __all__ = ("__version__", "__author__", "__author_email__", "__url__")
 
-__version__ = "0.7.4-onurgu"
+__version__ = "0.7.5"
 
 __author__ = 'Klaus Greff'
 __author_email__ = 'klaus.greff@startmail.com'

sacred/config/config_files.py

@@ -29,8 +29,12 @@ def __init__(self, load, dump, mode):
 
 yaml_extensions = ('.yaml', '.yml')
 if opt.has_yaml:
+    def load_yaml(filename):
+        return opt.yaml.load(filename, Loader=opt.yaml.FullLoader)
+    yaml_handler = Handler(load_yaml, opt.yaml.dump, '')
+
     for extension in yaml_extensions:
-        HANDLER_BY_EXT[extension] = Handler(opt.yaml.load, opt.yaml.dump, '')
+        HANDLER_BY_EXT[extension] = yaml_handler
 
 
 def get_handler(filename):

sacred/config/custom_containers.py

@@ -3,7 +3,7 @@
 from __future__ import division, print_function, unicode_literals
 
 import sacred.optional as opt
-from sacred.utils import join_paths
+from sacred.utils import join_paths, SacredError
 
 
 class FallbackDict(dict):
@@ -226,6 +226,80 @@ def revelation(self):
         return set()
 
 
+class ReadOnlyContainer:
+    def _readonly(self, *args, **kwargs):
+        raise SacredError(
+            self.message,
+            filter_traceback='always'
+        )
+
+
+class ReadOnlyDict(dict, ReadOnlyContainer):
+    """
+    A read-only variant of a `dict`
+    """
+    # Overwrite all methods that can modify a dict
+    clear = ReadOnlyContainer._readonly
+    pop = ReadOnlyContainer._readonly
+    popitem = ReadOnlyContainer._readonly
+    setdefault = ReadOnlyContainer._readonly
+    update = ReadOnlyContainer._readonly
+    __setitem__ = ReadOnlyContainer._readonly
+    __delitem__ = ReadOnlyContainer._readonly
+
+    def __init__(self, *args, **kwargs):
+        # Python 2.7 compatibility
+        self.message = kwargs.pop('message', None) or \
+            'This ReadOnlyDict is read-only!'
+
+        # Call dict init
+        super(ReadOnlyDict, self).__init__(*args, **kwargs)
+
+
+class ReadOnlyList(list, ReadOnlyContainer):
+    """
+    A read-only variant of a `list`
+    """
+    append = ReadOnlyContainer._readonly
+    clear = ReadOnlyContainer._readonly
+    extend = ReadOnlyContainer._readonly
+    insert = ReadOnlyContainer._readonly
+    pop = ReadOnlyContainer._readonly
+    remove = ReadOnlyContainer._readonly
+    reverse = ReadOnlyContainer._readonly
+    sort = ReadOnlyContainer._readonly
+    __setitem__ = ReadOnlyContainer._readonly
+    __delitem__ = ReadOnlyContainer._readonly
+
+    def __init__(self, *iterable, **kwargs):
+        # Python 2.7 compatibility
+        self.message = kwargs.pop('message', None) or \
+            'This ReadOnlyList is read-only!'
+
+        # Call list init
+        super(ReadOnlyList, self).__init__(*iterable)
+
+
+def make_read_only(o, error_message=None):
+    """
+    Converts every `list` and `dict` into `ReadOnlyList` and `ReadOnlyDict` in
+    a nested structure of `list`s, `dict`s and `tuple`s. Does not modify `o`
+    but returns the converted structure.
+    """
+    if isinstance(o, dict):
+        return ReadOnlyDict(
+            {k: make_read_only(v, error_message) for k, v in o.items()},
+            message=error_message)
+    elif isinstance(o, list):
+        return ReadOnlyList(
+            [make_read_only(v, error_message) for v in o],
+            message=error_message)
+    elif isinstance(o, tuple):
+        return tuple(map(make_read_only, o))
+    else:
+        return o
+
+
 SIMPLIFY_TYPE = {
     type(None): type(None),
     bool: bool,

sacred/config/utils.py

@@ -68,7 +68,7 @@ def assert_is_valid_key(key):
 def normalize_numpy(obj):
     if opt.has_numpy and isinstance(obj, opt.np.generic):
         try:
-            return opt.np.asscalar(obj)
+            return obj.item()
         except ValueError:
             pass
     return obj

sacred/initialize.py

@@ -9,6 +9,7 @@
 from sacred.config import (ConfigDict, chain_evaluate_config_scopes, dogmatize,
                            load_config_file, undogmatize)
 from sacred.config.config_summary import ConfigSummary
+from sacred.config.custom_containers import make_read_only
 from sacred.host_info import get_host_info
 from sacred.randomness import create_rnd, get_seed
 from sacred.run import Run
@@ -17,6 +18,7 @@
                           iterate_flattened, set_by_dotted_path,
                           recursive_update, iter_prefixes, join_paths,
                           NamedConfigNotFoundError, ConfigAddedError)
+from sacred.settings import SETTINGS
 
 
 class Scaffold(object):
@@ -162,12 +164,20 @@ def finalize_initialization(self, run):
         self.rnd = create_rnd(self.seed)
 
         for cfunc in self._captured_functions:
+            # Setup the captured function
             cfunc.logger = self.logger.getChild(cfunc.__name__)
-            cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix,
-                                              default={})
             seed = get_seed(self.rnd)
             cfunc.rnd = create_rnd(seed)
             cfunc.run = run
+            cfunc.config = get_by_dotted_path(self.get_fixture(), cfunc.prefix,
+                                              default={})
+
+            # Make configuration read only if enabled in settings
+            if SETTINGS.CONFIG.READ_ONLY_CONFIG:
+                cfunc.config = make_read_only(
+                    cfunc.config,
+                    error_message='The configuration is read-only in a '
+                                  'captured function!')
 
         if not run.force:
             self._warn_about_suspicious_changes()

sacred/metrics_logger.py

@@ -43,9 +43,9 @@ def log_scalar_metric(self, metric_name, value, step=None):
         if opt.has_numpy:
             np = opt.np
             if isinstance(value, np.generic):
-                value = np.asscalar(value)
+                value = value.item()
             if isinstance(step, np.generic):
-                step = np.asscalar(step)
+                step = step.item()
         if step is None:
             step = self._metric_step_counter.get(metric_name, -1) + 1
         self._logged_metrics.put(