Configuration management: Allow for creating / saving / loading configuration on the fly.

README.rst

@@ -79,15 +79,41 @@ Getting started
   automatically on-demand after the ECMWF
   `cdsapi<https://cds.climate.copernicus.eu/api-how-to>` client is 
   properly installed)
-* Adjust the `atlite/config.py <atlite/config.py>`_ directory paths to
-  point to the directory where you downloaded the dataset
 * Create a cutout, i.e. a geographical rectangle and a selection of
   times, e.g. all hours in 2011 and 2012, to narrow down the scope -
   see `examples/create_cutout.py <examples/create_cutout.py>`_
 * Select a sparse matrix of the geographical points inside the cutout
   you want to aggregate for your time series, and pass it to the
   appropriate converter function - see `examples/ <examples/>`_
 
+Optional: Configuration
+=======================
+
+Instead of manually providing configuration of data directories to function calls,
+you can create one or more configuration files to hold a standard configuration or
+custom configurations for each project.
+
+To create a standard configuration:
+
+* Configuration can be accessed and changed within your code using `atlite.config.<config_variable>`
+
+* To list all `configuration_variables` currently in use `print(atlite.config.ATTRS)`
+
+* Create a new directory `.atlite` in your home directory and place a `config.yaml` file there.
+  On unix systems this is `~/.atlite/config.yaml`,
+  on windows systems it usually is `C:\\Users\\\<Your Username\>\\.atlite\\config.yaml`.
+
+* Copy the settings and format of `atlite/default.config.yaml <atlite/default.config.yaml>`
+  and point the directories to where you downloaded or provided the respective data.
+  This file is automatically loaded when `import atlite`.
+
+* A specific configuration file can be loaded anytime within your code using
+  `atlite.config.read(<path to your configuration file>)`
+
+* A specific configuration can be stored anytime from within your code using
+  `atlite.config.save(<path to your configuration file>)`
+
+
 Licence
 =======
 

atlite/config.default.yaml

@@ -0,0 +1,19 @@
+# -------------------*DEFAULT SETTINGS*------------------- #
+# --------------------*DO NOT CHANGE*--------------------- #
+# For custom settings, see the 'config.example.yaml' file. #
+# --------------------*DO NOT CHANGE*--------------------- #
+
+# Folder for storing prepared cutout files
+cutout_dir: <ATLITE>/cutouts
+
+# Folder containing raw dataset data
+gebco_path: <ATLITE>/data/gebco
+ncep_dir:   <ATLITE>/data/ncep
+cordex_dir: <ATLITE>/data/cordex
+sarah_dir:  <ATLITE>/data/sarah
+
+# Folder for different wind turbine configuration files
+windturbine_dir: <ATLITE>/resources/windturbine
+
+# Folder for different solar panel configuration files
+solarpanel_dir: <ATLITE>/resources/solarpanel

atlite/config.example.yaml

@@ -0,0 +1,35 @@
+# Exemplary Settings for atlite
+
+
+## How to use this file.
+# 1. Copy this file into your home directory, usually that is
+#    "~" for linux users and "C:\Users\<Your Username>" for windows users.
+# 2. Rename the file to ".atlite.config.yaml" (note the trailing dot).
+# 3. Uncomment any setting you want to differ from the default settings.
+
+## Remarks
+# * Relative paths:
+#   Are relative to the location of the currently loaded config file,
+#   e.g. if the config file is located in your home directory,
+#   then any relative paths are considered relative to the homedirectory.
+#   Exception: Relative paths starting with "ATLITE/" (case-sensitive) are
+#   considered relative to the atlite-package directory.
+# * Custom location:
+#   You can use this configuration file and place it at custom locations with
+#   a custom file name. In this case you need to manually load the configuration
+#   after importing atlite with "atlite.config.read(<path with filename>)".
+
+# Folder for storing prepared cutout files
+# cutout_dir: <ATLITE>/cutouts
+
+# Folder containing raw dataset data
+# gebco_path: <ATLITE>/data/gebco
+# ncep_dir:   <ATLITE>/data/ncep
+# cordex_dir: <ATLITE>/data/cordex
+# sarah_dir:  <ATLITE>/data/sarah
+
+# Folder for different wind turbine configuration files
+# windturbine_dir: <ATLITE>/resources/windturbine
+
+# Folder for different solar panel configuration files
+# solarpanel_dir: <ATLITE>/resources/solarpanel

atlite/config.py

@@ -0,0 +1,120 @@
+## Copyright 2019 Johannes Hampp (Justus-Liebig University Giessen)
+
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 3 of the
+## License, or (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import pkg_resources
+import yaml
+import logging
+logger = logging.getLogger(__name__)
+
+
+_FILE_NAME = ".atlite.config.yaml"
+_FILE_SEARCH_PATH = os.path.join(os.path.expanduser("~"), _FILE_NAME)
+_DEFAULT_FILE_NAME = "config.default.yaml"
+_DEFAULT_SEARCH_PATH = pkg_resources.resource_filename(__name__, _DEFAULT_FILE_NAME)
+
+# List of all supported attributes for the config
+ATTRS = []
+
+# Implemented attributes
+cutout_dir = None
+windturbine_dir = None
+solarpanel_dir = None
+ncep_dir = None
+cordex_dir = None
+sarah_dir = None
+
+# Path of the configuration file.
+# Automatically updated when using provided API.
+config_path = ""
+
+def read(path):
+    """Read and set the configuration based on the file in 'path'."""
+
+    if not os.path.isfile(path):
+        raise TypeError("Invalid configuration file path: "
+                        "{p}".format(p=path))
+
+    with open(path, "r") as config_file:
+        config_dict = yaml.safe_load(config_file)
+
+    config_dict['config_path'] = path
+    update(config_dict)
+
+    logger.info("Configuration from {p} successfully read.".format(p=path))
+
+def save(path, overwrite=False):
+    """Write the current configuration into a config file in the specified path.
+
+    Parameters
+    ----------
+    path : string or os.path
+        Including name of the new config file.
+    overwrite : boolean
+        (Default: False) Allow overwriting of existing files.
+
+    """
+
+    if os.path.exists(path) and overwrite is False:
+        raise FileExistsError("Overwriting disallowed for {p}".format(p=path))
+
+    # New path now points to the current config
+    global config_path
+    config_path = path
+
+    # Construct attribute dict
+    global ATTRS
+    _update_variables()
+
+    config = {key:globals()[key] for key in ATTRS}
+
+    with open(path, "w") as config_file:
+        yaml.dump(config, config_file, default_flow_style=False)
+
+def update(config_dict):
+    """Update the existing config based on the `config_dict` dictionary; resets `config_path`."""
+
+    globals().update(config_dict)
+    _update_variables()
+
+def reset():
+    """Reset the configuration to its initial values."""
+
+    # Test for file existence in order to not try to read
+    # non-existing configuration files at this point (do not confuse the user)
+    for path in [_DEFAULT_SEARCH_PATH, _FILE_SEARCH_PATH]:
+        if os.path.isfile(path):
+            read(path)
+
+    # Notify user of empty config
+    if not config_path:
+        logger.warn("No valid configuration file found in default and home directories. "
+                    "No configuration is loaded, manual configuration required.")
+
+def _update_variables():
+    """Update list of provided attributes by the module."""
+
+    global ATTRS
+
+    ATTRS = {k for k,v in globals().items() if not k.startswith("_") and not callable(v)}
+
+    # Manually remove imported modules and the attribute itself from the list
+    ATTRS = ATTRS - {"ATTRS", "logging",
+                     "logger", "os", "pkg_resources", "yaml"}
+
+
+# Load the configuration at first module import
+reset()

atlite/cutout.py

@@ -31,6 +31,7 @@
 import logging
 logger = logging.getLogger(__name__)
 
+from . import config
 
 from . import datasets, utils
 
@@ -42,14 +43,15 @@
 class Cutout(object):
     dataset_module = None
 
-    def __init__(self, name=None, data=None, cutout_dir=".", **cutoutparams):
+    def __init__(self, name=None, data=None, cutout_dir=None, **cutoutparams):
+
         if isinstance(name, xr.Dataset):
             data = name
             name = data.attrs.get("name", "unnamed")
 
-        if '/' in name:
-            cutout_dir, name = os.path.split(name)
-
+        if not cutout_dir:
+            config.cutout_dir
+        
         self.name = name
         self.cutout_dir = cutout_dir
 

atlite/datasets/cordex.py

@@ -32,7 +32,7 @@
 import os
 import glob
 
-from ..config import cordex_dir
+from .. import config
 from ..gis import RotProj
 
 # Model and Projection Settings
@@ -121,41 +121,41 @@ def tasks_yearly_cordex(xs, ys, yearmonths, prepare_func, template, oldname, new
     'influx': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='rsds', newname='influx',
-                   template=os.path.join(cordex_dir, '{model}', 'influx', 'rsds_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'influx', 'rsds_*_{year}*.nc')),
     'outflux': dict(tasks_func=tasks_yearly_cordex,
                     prepare_func=prepare_data_cordex,
                     oldname='rsus', newname='outflux',
-                    template=os.path.join(cordex_dir, '{model}', 'outflux', 'rsus_*_{year}*.nc')),
+                    template=os.path.join(config.cordex_dir, '{model}', 'outflux', 'rsus_*_{year}*.nc')),
     'temperature': dict(tasks_func=tasks_yearly_cordex,
                         prepare_func=prepare_data_cordex,
                         oldname='tas', newname='temperature',
-                        template=os.path.join(cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc')),
+                        template=os.path.join(config.cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc')),
     'humidity': dict(tasks_func=tasks_yearly_cordex,
                         prepare_func=prepare_data_cordex,
                         oldname='hurs', newname='humidity',
-                        template=os.path.join(cordex_dir, '{model}', 'humidity', 'hurs_*_{year}*.nc')),
+                        template=os.path.join(config.cordex_dir, '{model}', 'humidity', 'hurs_*_{year}*.nc')),
     'wnd10m': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='sfcWind', newname='wnd10m',
-                   template=os.path.join(cordex_dir, '{model}', 'wind', 'sfcWind_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'wind', 'sfcWind_*_{year}*.nc')),
     'roughness': dict(tasks_func=tasks_yearly_cordex,
                       prepare_func=prepare_static_data_cordex,
                       oldname='rlst', newname='roughness',
-                      template=os.path.join(cordex_dir, '{model}', 'roughness', 'rlst_*.nc')),
+                      template=os.path.join(config.cordex_dir, '{model}', 'roughness', 'rlst_*.nc')),
     'runoff': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='mrro', newname='runoff',
-                   template=os.path.join(cordex_dir, '{model}', 'runoff', 'mrro_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'runoff', 'mrro_*_{year}*.nc')),
     'height': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_static_data_cordex,
                    oldname='orog', newname='height',
-                   template=os.path.join(cordex_dir, '{model}', 'altitude', 'orog_*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'altitude', 'orog_*.nc')),
     'CWT': dict(tasks_func=tasks_yearly_cordex,
                 prepare_func=prepare_weather_types_cordex,
                 oldname='CWT', newname='CWT',
-                template=os.path.join(cordex_dir, '{model}', 'weather_types', 'CWT_*_{year}*.nc')),
+                template=os.path.join(config.cordex_dir, '{model}', 'weather_types', 'CWT_*_{year}*.nc')),
 }
 
 meta_data_config = dict(prepare_func=prepare_meta_cordex,
-                        template=os.path.join(cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc'),
+                        template=os.path.join(config.cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc'),
                         height_config=weather_data_config['height'])

atlite/datasets/ncep.py

@@ -31,7 +31,7 @@
 import subprocess
 import shutil
 
-from ..config import ncep_dir
+from .. import config
 
 engine = 'pynio'
 projection = 'latlong'
@@ -222,30 +222,30 @@ def tasks_height_ncep(xs, ys, yearmonths, prepare_func, template, meta_attrs, **
 weather_data_config = {
     'influx': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_influx_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/dswsfc.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/dswsfc.*.grb2')),
     'outflux': dict(tasks_func=tasks_monthly_ncep,
                     prepare_func=prepare_outflux_ncep,
-                    template=os.path.join(ncep_dir, '{year}{month:0>2}/uswsfc.*.grb2')),
+                    template=os.path.join(config.ncep_dir, '{year}{month:0>2}/uswsfc.*.grb2')),
     'temperature': dict(tasks_func=tasks_monthly_ncep,
                         prepare_func=prepare_temperature_ncep,
-                        template=os.path.join(ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2')),
+                        template=os.path.join(config.ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2')),
     'soil temperature': dict(tasks_func=tasks_monthly_ncep,
                              prepare_func=prepare_soil_temperature_ncep,
-                             template=os.path.join(ncep_dir, '{year}{month:0>2}/soilt1.*.grb2')),
+                             template=os.path.join(config.ncep_dir, '{year}{month:0>2}/soilt1.*.grb2')),
     'wnd10m': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_wnd10m_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/wnd10m.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/wnd10m.*.grb2')),
     'runoff': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_runoff_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/runoff.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/runoff.*.grb2')),
     'roughness': dict(tasks_func=tasks_monthly_ncep,
                       prepare_func=prepare_roughness_ncep,
-                      template=os.path.join(ncep_dir, '{year}{month:0>2}/flxf.gdas.*.grb2')),
+                      template=os.path.join(config.ncep_dir, '{year}{month:0>2}/flxf.gdas.*.grb2')),
     'height': dict(tasks_func=tasks_height_ncep,
                    prepare_func=prepare_height_ncep,
-                   template=os.path.join(ncep_dir, 'height/cdas1.20130101.splgrbanl.grb2'))
+                   template=os.path.join(config.ncep_dir, 'height/cdas1.20130101.splgrbanl.grb2'))
 }
 
 meta_data_config = dict(prepare_func=prepare_meta_ncep,
-                        template=os.path.join(ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2'),
+                        template=os.path.join(config.ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2'),
                         height_config=weather_data_config['height'])

atlite/datasets/sarah.py

@@ -27,6 +27,9 @@
 import xarray as xr
 from functools import partial
 import glob
+
+from .. import config
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -86,7 +89,7 @@ def p(s):
         return ds.load()
 
 def get_data(coords, date, feature, x, y, **creation_parameters):
-    sis_fn, sid_fn = _get_filenames(creation_parameters.get('sarah_dir', sarah_dir), date)
+    sis_fn, sid_fn = _get_filenames(creation_parameters.get('sarah_dir', config.sarah_dir), date)
     res = creation_parameters.get('resolution', resolution)
 
     with xr.open_mfdataset(sis_fn) as ds_sis, \