Configuration management: Allow for creating / saving / loading configuration on the fly.

README.rst

@@ -79,15 +79,41 @@ Getting started
   automatically on-demand after the ECMWF
   `cdsapi<https://cds.climate.copernicus.eu/api-how-to>` client is 
   properly installed)
-* Adjust the `atlite/config.py <atlite/config.py>`_ directory paths to
-  point to the directory where you downloaded the dataset
 * Create a cutout, i.e. a geographical rectangle and a selection of
   times, e.g. all hours in 2011 and 2012, to narrow down the scope -
   see `examples/create_cutout.py <examples/create_cutout.py>`_
 * Select a sparse matrix of the geographical points inside the cutout
   you want to aggregate for your time series, and pass it to the
   appropriate converter function - see `examples/ <examples/>`_
 
+Optional: Configuration
+=======================
+
+Instead of manually providing configuration of data directories to function calls,
+you can create one or more configuration files to hold a standard configuration or
+custom configurations for each project.
+
+To create a standard configuration:
+
+* Configuration can be accessed and changed within your code using `atlite.config.<config_variable>`
+
+* To list all `configuration_variables` currently in use `print(atlite.config.ATTRS)`
+
+* Create a new directory `.atlite` in your home directory and place a `config.yaml` file there.
+  On unix systems this is `~/.atlite/config.yaml`,
+  on windows systems it usually is `C:\\Users\\\<Your Username\>\\.atlite\\config.yaml`.
+
+* Copy the settings and format of `atlite/default.config.yaml <atlite/default.config.yaml>`
+  and point the directories to where you downloaded or provided the respective data.
+  This file is automatically loaded when `import atlite`.
+
+* A specific configuration file can be loaded anytime within your code using
+  `atlite.config.read(<path to your configuration file>)`
+
+* A specific configuration can be stored anytime from within your code using
+  `atlite.config.save(<path to your configuration file>)`
+
+
 Licence
 =======
 

atlite/config.py

@@ -0,0 +1,124 @@
+## Copyright 2019 Johannes Hampp (Justus-Liebig University Giessen)
+
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 3 of the
+## License, or (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import pkg_resources
+import yaml
+import logging
+logger = logging.getLogger(__name__)
+
+
+_FILE_NAME = "config.yaml"
+_DEFAULT_FILE_NAME = "default.config.yaml"
+
+# Search paths for the yaml-configuration file
+_SEARCH_PATHS = (
+    # User home directory - Custom
+    os.path.join(os.path.expanduser("~"), ".atlite", _FILE_NAME),
+    # Package install directory - Custom
+    pkg_resources.resource_filename(__name__, _FILE_NAME),
+    # Package install directory - Default
+    pkg_resources.resource_filename(__name__, _DEFAULT_FILE_NAME)
+)
+
+# List of all supported attributes for the config
+ATTRS = []
+
+# Implemented attributes
+cutout_dir = None
+windturbine_dir = None
+solarpanel_dir = None
+ncep_dir = None
+cordex_dir = None
+sarah_dir = None
+
+# Path of the configuration file.
+# Automatically updated when using provided API.
+config_path = None
+
+def read(path):
+    """Read and set the configuration based on the file in 'path'."""
+
+    if not os.path.isfile(path):
+        raise TypeError("Invalid configuration file path: "
+                        "{p}".format(p=path))
+
+    with open(path, "r") as config_file:
+        config_dict = yaml.safe_load(config_file)
+
+    config_dict.update({"config_path": path})
+    update(config_dict)
+
+    logger.info("Configuration from {p} successfully read.".format(p=path))
+
+def save(path, overwrite=False):
+    """Write the current configuration into a config file in the specified path.
+
+    Parameters
+    ----------
+    path : string or os.path
+        Including name of the new config file.
+    overwrite : boolean
+        (Default: False) Allow overwriting of existing files.
+
+    """
+
+    if os.path.exists(path) and overwrite is False:
+        raise FileExistsError("Overwriting disallowed for {p}".format(p=path))
+
+    # New path now points to the current config
+    global config_path
+    config_path = path
+
+    # Construct attribute dict
+    global ATTRS
+    _update_variables()
+
+    config = {key:globals()[key] for key in ATTRS}
+
+    with open(path, "w") as config_file:
+        yaml.dump(config, config_file, default_flow_style=False)
+
+def update(config_dict):
+    """Update the existing config based on the `config_dict` dictionary; resets `config_path`."""
+
+    globals().update(config_dict)
+    _update_variables()
+
+def _update_variables():
+    """Update list of provided attributes by the module."""
+
+    global ATTRS
+
+    ATTRS = {k for k,v in globals().items() if not k.startswith("_") and not callable(v)}
+
+    # Manually remove imported modules and the attribute itself from the list
+    ATTRS = ATTRS - {"ATTRS", "logging",
+                     "logger", "os", "pkg_resources", "yaml"}
+
+
+# Try to load configuration from standard paths
+for path in _SEARCH_PATHS:
+    if os.path.isfile(path):
+        # Don't check if the file is actually what it claims to be
+        # also: consider a read without error a success.
+        read(path)
+        break
+
+# Notify user of empty config
+if not config_path:
+    logger.warn("No valid configuration file found in default paths. "
+                "No configuration is loaded, manual configuration required.")

atlite/cutout.py

@@ -31,6 +31,7 @@
 import logging
 logger = logging.getLogger(__name__)
 
+from . import config
 
 from . import datasets, utils
 
@@ -42,16 +43,21 @@
 class Cutout(object):
     dataset_module = None
 
-    def __init__(self, name=None, data=None, cutout_dir=".", **cutoutparams):
+    def __init__(self, name=None, data=None, cutout_dir=None, **cutoutparams):
+
         if isinstance(name, xr.Dataset):
             data = name
             name = data.attrs.get("name", "unnamed")
 
-        if '/' in name:
-            cutout_dir, name = os.path.split(name)
+        if name:
+            cutout_dir = os.path.dirname(name)
+            name = os.path.basename(name)
+
+            self.cutout_dir = cutout_dir
+            self.name = name
 
-        self.name = name
-        self.cutout_dir = cutout_dir
+        if not cutout_dir:
+            config.cutout_dir
 
         if 'bounds' in cutoutparams:
             x1, y1, x2, y2 = cutoutparams.pop('bounds')

atlite/datasets/cordex.py

@@ -32,7 +32,7 @@
 import os
 import glob
 
-from ..config import cordex_dir
+from .. import config
 from ..gis import RotProj
 
 # Model and Projection Settings
@@ -121,41 +121,41 @@ def tasks_yearly_cordex(xs, ys, yearmonths, prepare_func, template, oldname, new
     'influx': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='rsds', newname='influx',
-                   template=os.path.join(cordex_dir, '{model}', 'influx', 'rsds_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'influx', 'rsds_*_{year}*.nc')),
     'outflux': dict(tasks_func=tasks_yearly_cordex,
                     prepare_func=prepare_data_cordex,
                     oldname='rsus', newname='outflux',
-                    template=os.path.join(cordex_dir, '{model}', 'outflux', 'rsus_*_{year}*.nc')),
+                    template=os.path.join(config.cordex_dir, '{model}', 'outflux', 'rsus_*_{year}*.nc')),
     'temperature': dict(tasks_func=tasks_yearly_cordex,
                         prepare_func=prepare_data_cordex,
                         oldname='tas', newname='temperature',
-                        template=os.path.join(cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc')),
+                        template=os.path.join(config.cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc')),
     'humidity': dict(tasks_func=tasks_yearly_cordex,
                         prepare_func=prepare_data_cordex,
                         oldname='hurs', newname='humidity',
-                        template=os.path.join(cordex_dir, '{model}', 'humidity', 'hurs_*_{year}*.nc')),
+                        template=os.path.join(config.cordex_dir, '{model}', 'humidity', 'hurs_*_{year}*.nc')),
     'wnd10m': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='sfcWind', newname='wnd10m',
-                   template=os.path.join(cordex_dir, '{model}', 'wind', 'sfcWind_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'wind', 'sfcWind_*_{year}*.nc')),
     'roughness': dict(tasks_func=tasks_yearly_cordex,
                       prepare_func=prepare_static_data_cordex,
                       oldname='rlst', newname='roughness',
-                      template=os.path.join(cordex_dir, '{model}', 'roughness', 'rlst_*.nc')),
+                      template=os.path.join(config.cordex_dir, '{model}', 'roughness', 'rlst_*.nc')),
     'runoff': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_data_cordex,
                    oldname='mrro', newname='runoff',
-                   template=os.path.join(cordex_dir, '{model}', 'runoff', 'mrro_*_{year}*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'runoff', 'mrro_*_{year}*.nc')),
     'height': dict(tasks_func=tasks_yearly_cordex,
                    prepare_func=prepare_static_data_cordex,
                    oldname='orog', newname='height',
-                   template=os.path.join(cordex_dir, '{model}', 'altitude', 'orog_*.nc')),
+                   template=os.path.join(config.cordex_dir, '{model}', 'altitude', 'orog_*.nc')),
     'CWT': dict(tasks_func=tasks_yearly_cordex,
                 prepare_func=prepare_weather_types_cordex,
                 oldname='CWT', newname='CWT',
-                template=os.path.join(cordex_dir, '{model}', 'weather_types', 'CWT_*_{year}*.nc')),
+                template=os.path.join(config.cordex_dir, '{model}', 'weather_types', 'CWT_*_{year}*.nc')),
 }
 
 meta_data_config = dict(prepare_func=prepare_meta_cordex,
-                        template=os.path.join(cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc'),
+                        template=os.path.join(config.cordex_dir, '{model}', 'temperature', 'tas_*_{year}*.nc'),
                         height_config=weather_data_config['height'])

atlite/datasets/ncep.py

@@ -31,7 +31,7 @@
 import subprocess
 import shutil
 
-from ..config import ncep_dir
+from .. import config
 
 engine = 'pynio'
 projection = 'latlong'
@@ -222,30 +222,30 @@ def tasks_height_ncep(xs, ys, yearmonths, prepare_func, template, meta_attrs, **
 weather_data_config = {
     'influx': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_influx_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/dswsfc.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/dswsfc.*.grb2')),
     'outflux': dict(tasks_func=tasks_monthly_ncep,
                     prepare_func=prepare_outflux_ncep,
-                    template=os.path.join(ncep_dir, '{year}{month:0>2}/uswsfc.*.grb2')),
+                    template=os.path.join(config.ncep_dir, '{year}{month:0>2}/uswsfc.*.grb2')),
     'temperature': dict(tasks_func=tasks_monthly_ncep,
                         prepare_func=prepare_temperature_ncep,
-                        template=os.path.join(ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2')),
+                        template=os.path.join(config.ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2')),
     'soil temperature': dict(tasks_func=tasks_monthly_ncep,
                              prepare_func=prepare_soil_temperature_ncep,
-                             template=os.path.join(ncep_dir, '{year}{month:0>2}/soilt1.*.grb2')),
+                             template=os.path.join(config.ncep_dir, '{year}{month:0>2}/soilt1.*.grb2')),
     'wnd10m': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_wnd10m_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/wnd10m.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/wnd10m.*.grb2')),
     'runoff': dict(tasks_func=tasks_monthly_ncep,
                    prepare_func=prepare_runoff_ncep,
-                   template=os.path.join(ncep_dir, '{year}{month:0>2}/runoff.*.grb2')),
+                   template=os.path.join(config.ncep_dir, '{year}{month:0>2}/runoff.*.grb2')),
     'roughness': dict(tasks_func=tasks_monthly_ncep,
                       prepare_func=prepare_roughness_ncep,
-                      template=os.path.join(ncep_dir, '{year}{month:0>2}/flxf.gdas.*.grb2')),
+                      template=os.path.join(config.ncep_dir, '{year}{month:0>2}/flxf.gdas.*.grb2')),
     'height': dict(tasks_func=tasks_height_ncep,
                    prepare_func=prepare_height_ncep,
-                   template=os.path.join(ncep_dir, 'height/cdas1.20130101.splgrbanl.grb2'))
+                   template=os.path.join(config.ncep_dir, 'height/cdas1.20130101.splgrbanl.grb2'))
 }
 
 meta_data_config = dict(prepare_func=prepare_meta_ncep,
-                        template=os.path.join(ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2'),
+                        template=os.path.join(config.ncep_dir, '{year}{month:0>2}/tmp2m.*.grb2'),
                         height_config=weather_data_config['height'])

atlite/datasets/sarah.py

@@ -27,6 +27,9 @@
 import xarray as xr
 from functools import partial
 import glob
+
+from .. import config
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -86,7 +89,7 @@ def p(s):
         return ds.load()
 
 def get_data(coords, date, feature, x, y, **creation_parameters):
-    sis_fn, sid_fn = _get_filenames(creation_parameters.get('sarah_dir', sarah_dir), date)
+    sis_fn, sid_fn = _get_filenames(creation_parameters.get('sarah_dir', config.sarah_dir), date)
     res = creation_parameters.get('resolution', resolution)
 
     with xr.open_mfdataset(sis_fn) as ds_sis, \

atlite/default.config.yaml

@@ -0,0 +1,25 @@
+# Default Settings
+# For ATLITE - the Renewable Energy Atlas in its lite version.
+
+# TO OVERWRITE THESE DEFAULT SETTINGS:
+# copy the file in this directory or into a folder ".atlite" in your user's home
+# directory (Linux: "~/.atlite", Windows: usually "C:\Users\<Your Username>\.atlite\")
+# then rename it to '<your home directory>/.atlite/config.yaml'.
+
+
+# NOTE
+# Relative location paths in this file are considered to be relative
+# to the location of this configuration file, i.e.
+# if this file in the path ~/.atlite/config.yaml ,
+# then relative paths should be considered relative to ~/.atlite/
+# Alternatively: One can use absolute paths.
+
+# Dataset Settings
+gebco_path: data/gebco
+cutout_dir: cutouts
+ncep_dir:   data/ncep
+cordex_dir: data/cordex
+sarah_dir:  data/sarah
+
+windturbine_dir: resources/windturbine
+solarpanel_dir: resources/solarpanel