Solved issues in configuration files

doc/quickstart/configure.rst

@@ -42,90 +42,95 @@ If the option ``--path`` is omitted, the file will be created in
 ``${HOME}/.esmvaltool``
 
 The following shows the default settings from the ``config-user.yml`` file
-with explanations in a commented line above each option:
+with explanations in a commented line above each option. If only certain values
+are allowed for an option, these are listed after ``---``. The option in square
+brackets is the default value, i.e., the one that is used if this option is
+omitted in the file.
 
 .. code-block:: yaml
 
   # Destination directory where all output will be written
-  # including log files and performance stats
+  # Includes log files and performance stats.
   output_dir: ./esmvaltool_output
 
   # Directory for storing downloaded climate data
   download_dir: ~/climate_data
 
+  # Disable automatic downloads --- [true]/false
   # Disable the automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX,
-  # and obs4MIPs data from ESGF [true]/false.
+  # and obs4MIPs data from ESGF by default. This is useful if you are working
+  # on a computer without an internet connection.
   offline: true
 
-  # Auxiliary data directory, used by some recipes to look for additional datasets
+  # Auxiliary data directory
+  # Used by some recipes to look for additional datasets.
   auxiliary_data_dir: ~/auxiliary_data
 
-  # Rootpaths to the data from different projects (lists are also possible)
-  # these are generic entries to better allow you to enter your own
-  # For site-specific entries, see the default config-user.yml file
-  # that can be installed with the command `esmvaltool config get_config_user`.
-  # For each project, this can be either a single path or a list of paths.
-  # Comment out these when using a site-specific path
+  # Rootpaths to the data from different projects
+  # This default setting will work if automatic downloads are enabled via
+  # ``offline=False``. Lists are also possible. For site-specific entries,
+  # see the default ``config-user.yml`` file that can be installed with the
+  # command ``esmvaltool config get_config_user``. For each project, this can
+  # be either a single path or a list of paths. Comment out these when using a
+  # site-specific path.
   rootpath:
     default: ~/climate_data
 
-  # Directory structure for input data: [default]/ESGF/BADC/DKRZ/ETHZ/etc
-  # See config-developer.yml for definitions.
-  # comment out/replace as per needed
+  # Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc.
+  # This default setting will work if automatic downloads are enabled via
+  # ``offline=False``. See ``config-developer.yml`` for definitions. Comment
+  # out/replace as per needed.
   drs:
     CMIP3: ESGF
     CMIP5: ESGF
     CMIP6: ESGF
     CORDEX: ESGF
     obs4MIPs: ESGF
 
-  # Run at most this many tasks in parallel [null]/1/2/3/4/..
-  # Set to null to use the number of available CPUs.
-  # If you run out of memory, try setting max_parallel_tasks to 1 and check the
-  # amount of memory you need for that by inspecting the file
-  # run/resource_usage.txt in the output directory. Using the number there you
-  # can increase the number of parallel tasks again to a reasonable number for
-  # the amount of memory available in your system.
+  # Run at most this many tasks in parallel --- [null]/1/2/3/4/...
+  # Set to ``null`` to use the number of available CPUs. If you run out of
+  # memory, try setting max_parallel_tasks to ``1`` and check the amount of
+  # memory you need for that by inspecting the file ``run/resource_usage.txt`` in
+  # the output directory. Using the number there you can increase the number of
+  # parallel tasks again to a reasonable number for the amount of memory
+  # available in your system.
   max_parallel_tasks: null
 
-  # Set the console log level debug, [info], warning, error
-  # for much more information printed to screen set log_level: debug
+  # Log level of the console --- debug/[info]/warning/error
+  # For much more information printed to screen set log_level to ``debug``.
   log_level: info
 
-  # Exit on warning (only for NCL diagnostic scripts)? true/[false]
+  # Exit on warning --- true/[false]
+  # Only used in NCL diagnostic scripts.
   exit_on_warning: false
 
-  # Plot file format? [png]/pdf/ps/eps/epsi
+  # Plot file format --- [png]/pdf/ps/eps/epsi
   output_file_type: png
 
-  # Remove the ``preproc`` dir if the run was successful
-  # By default this option is set to "true", so all preprocessor output files
-  # will be removed after a successful run. Set to "false" if you need those files.
+  # Remove the ``preproc`` directory if the run was successful --- [true]/false
+  # By default this option is set to ``true``, so all preprocessor output files
+  # will be removed after a successful run. Set to ``false`` if you need those files.
   remove_preproc_dir: true
 
-  # Use netCDF compression true/[false]
+  # Use netCDF compression --- true/[false]
   compress_netcdf: false
 
-  # Save intermediary cubes in the preprocessor true/[false]
-  # set to true will save the output cube from each preprocessing step
-  # these files are numbered according to the preprocessing order
+  # Save intermediary cubes in the preprocessor --- true/[false]
+  # Setting this to ``true`` will save the output cube from each preprocessing
+  # step. These files are numbered according to the preprocessing order.
   save_intermediary_cubes: false
 
-  # Use a profiling tool for the diagnostic run [false]/true
+  # Use a profiling tool for the diagnostic run --- [false]/true
   # A profiler tells you which functions in your code take most time to run.
-  # For this purpose we use vprof, see below for notes
-  # Only available for Python diagnostics
+  # For this purpose we use ``vprof``, see below for notes. Only available for
+  # Python diagnostics.
   profile_diagnostic: false
 
-  # Path to custom config-developer file, to customise project configurations.
-  # See config-developer.yml for an example. Set to "null" to use the default
+  # Path to custom ``config-developer.yml`` file
+  # This can be used to customise project configurations. See
+  # ``config-developer.yml`` for an example. Set to ``null`` to use the default.
   config_developer_file: null
 
-.. code-block:: yaml
-
-  # Auxiliary data directory (used for some additional datasets)
-  auxiliary_data_dir: ~/auxiliary_data
-
 The ``offline`` setting can be used to disable or enable automatic downloads from ESGF.
 If ``offline`` is set to ``false``, the tool will automatically download
 any CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs data that is required to run a recipe

esmvalcore/_config/_config.py

@@ -104,7 +104,7 @@ def read_config_user_file(config_file, folder_name, options=None):
 
     # set defaults
     defaults = {
-        'auxiliary_data_dir': 'auxiliary_data',
+        'auxiliary_data_dir': '~/auxiliary_data',
         'compress_netcdf': False,
         'config_developer_file': None,
         'drs': {},
@@ -114,7 +114,7 @@ def read_config_user_file(config_file, folder_name, options=None):
         'max_parallel_tasks': None,
         'offline': True,
         'output_file_type': 'png',
-        'output_dir': 'esmvaltool_output',
+        'output_dir': '~/esmvaltool_output',
         'profile_diagnostic': False,
         'remove_preproc_dir': True,
         'resume_from': [],

esmvalcore/config-user.yml

@@ -5,7 +5,11 @@
 # Note for users:
 # --------------
 # Site-specific entries for different HPC centers are given at the bottom of
-# this file. Comment out/replace as needed.
+# this file. Comment out/replace as needed. This default version of the file
+# can be used in combination with the command line argument ``offline=False``
+# If only certain values are allowed for an option, these are listed after
+# ``---``. The option in square brackets is the default value, i.e., the one
+# that is used if this option is omitted in the file.
 #
 ###############################################################################
 #
@@ -18,84 +22,109 @@
 ###############################################################################
 ---
 
-# Directory where results are stored
+# Destination directory where all output will be written
+# Includes log files and performance stats.
 output_dir: ~/esmvaltool_output
 
 # Directory for storing downloaded climate data
 download_dir: ~/climate_data
 
-# Auxiliary data directory (used for some additional datasets)
+# Auxiliary data directory
+# Used by some recipes to look for additional datasets.
 auxiliary_data_dir: ~/auxiliary_data
 
-# Run at most this many tasks in parallel [null]/1/2/3/4/..
-# Set to null to use the number of available CPUs.
-# If you run out of memory, try setting max_parallel_tasks to 1 and check the
-# amount of memory you need for that by inspecting the file
-# run/resource_usage.txt in the output directory. Using the number there you
-# can increase the number of parallel tasks again to a reasonable number for
-# the amount of memory available in your system.
+# Run at most this many tasks in parallel --- [null]/1/2/3/4/...
+# Set to ``null`` to use the number of available CPUs. If you run out of
+# memory, try setting max_parallel_tasks to ``1`` and check the amount of
+# memory you need for that by inspecting the file ``run/resource_usage.txt`` in
+# the output directory. Using the number there you can increase the number of
+# parallel tasks again to a reasonable number for the amount of memory
+# available in your system.
 max_parallel_tasks: null
 
+# Disable automatic downloads --- [true]/false
 # Disable the automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX,
 # and obs4MIPs data from ESGF by default. This is useful if you are working
 # on a computer without an internet connection.
 offline: true
 
-# Log level of the console
+# Log level of the console --- debug/[info]/warning/error
+# For much more information printed to screen set log_level to ``debug``.
 log_level: info
 
-# Exit on warning (only for NCL diagnostic scripts)? true/[false]
+# Exit on warning --- true/[false]
+# # Only used in NCL diagnostic scripts.
 exit_on_warning: false
 
-# Plot file format? [png]/pdf/ps/eps/epsi
+# Plot file format --- [png]/pdf/ps/eps/epsi
 output_file_type: png
 
-# Remove the preproc dir if all fine
+# Remove the ``preproc`` directory if the run was successful --- [true]/false
+# By default this option is set to ``true``, so all preprocessor output files
+# will be removed after a successful run. Set to ``false`` if you need those
+# files.
 remove_preproc_dir: true
 
-# Use netCDF compression true/[false]
+# Use netCDF compression --- true/[false]
 compress_netcdf: false
 
-# Save intermediary cubes in the preprocessor true/[false]
+# Save intermediary cubes in the preprocessor --- true/[false]
+# Setting this to ``true`` will save the output cube from each preprocessing
+# step. These files are numbered according to the preprocessing order.
 save_intermediary_cubes: false
 
-# Path to custom config-developer file, to customise project configurations.
-# See config-developer.yml for an example. Set to None to use the default
+# Path to custom ``config-developer.yml`` file
+# This can be used to customise project configurations. See
+# ``config-developer.yml`` for an example. Set to ``null`` to use the default.
 config_developer_file: null
 
-# Get profiling information for diagnostics
-# Only available for Python diagnostics
+# Use a profiling tool for the diagnostic run --- [false]/true
+# A profiler tells you which functions in your code take most time to run.
+# Only available for Python diagnostics.
 profile_diagnostic: false
 
-# Rootpaths to the data from different projects (lists are also possible)
-# these are generic entries to better allow you to enter your own
-# For site-specific entries, see below
-# Comment out these when using a site-specific path
+# Rootpaths to the data from different projects
+# This default setting will work if automatic downloads are enabled via
+# ``offline=False``. Lists are also possible. For site-specific entries and
+# more examples, see below. Comment out these when using a site-specific path.
 rootpath:
-  CMIP3: [~/cmip3_inputpath1, ~/cmip3_inputpath2]
-  CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2]
-  CMIP6: [~/cmip6_inputpath1, ~/cmip6_inputpath2]
-  OBS: ~/obs_inputpath
-  OBS6: ~/obs6_inputpath
-  obs4MIPs: ~/obs4mips_inputpath
-  ana4mips: ~/ana4mips_inputpath
-  native6: ~/native6_inputpath
-  RAWOBS: ~/rawobs_inputpath
-  default: ~/default_inputpath
-
-# Directory structure for input data: [default]/ESGF/BADC/DKRZ/ETHZ/etc
-# See config-developer.yml for definitions.
-# comment out/replace as per needed
+  default: ~/climate_data
+
+# Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc.
+# This default setting will work if automatic downloads are enabled via
+# ``offline=False``. See ``config-developer.yml`` for definitions. Comment
+# out/replace as per needed.
 drs:
-  CMIP3: default
-  CMIP5: default
-  CMIP6: default
-  CORDEX: default
-  obs4MIPs: default
+  CMIP3: ESGF
+  CMIP5: ESGF
+  CMIP6: ESGF
+  CORDEX: ESGF
+  obs4MIPs: ESGF
+
+# Example rootpaths and directory structure that showcases the different
+# projects and also the use of lists
+# For site-specific entries, see below.
+#rootpath:
+#  CMIP3: [~/cmip3_inputpath1, ~/cmip3_inputpath2]
+#  CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2]
+#  CMIP6: [~/cmip6_inputpath1, ~/cmip6_inputpath2]
+#  OBS: ~/obs_inputpath
+#  OBS6: ~/obs6_inputpath
+#  obs4MIPs: ~/obs4mips_inputpath
+#  ana4mips: ~/ana4mips_inputpath
+#  native6: ~/native6_inputpath
+#  RAWOBS: ~/rawobs_inputpath
+#  default: ~/default_inputpath
+#drs:
+#  CMIP3: default
+#  CMIP5: default
+#  CMIP6: default
+#  CORDEX: default
+#  obs4MIPs: default
 
 # Directory tree created by automatically downloading from ESGF
 # Uncomment the lines below to locate data that has been automatically
-# downloaded from ESGF (using "offline: false")
+# downloaded from ESGF (using ``offline: false``).
 #rootpath:
 #  CMIP3: ~/climate_data
 #  CMIP5: ~/climate_data
@@ -109,8 +138,8 @@ drs:
 #  CORDEX: ESGF
 #  obs4MIPs: ESGF
 
-# Site-specific entries: Jasmin
-# Uncomment the lines below to locate data on JASMIN
+# Site-specific entries: JASMIN
+# Uncomment the lines below to locate data on JASMIN.
 #auxiliary_data_dir: /gws/nopw/j04/esmeval/aux_data/AUX
 #rootpath:
 #  CMIP6: /badc/cmip6/data/CMIP6
@@ -132,8 +161,8 @@ drs:
 #  ana4mips: default
 
 # Site-specific entries: DKRZ
-# Uncomment the lines below to locate data on Mistral at DKRZ
-# Note: Use "offline: false" only on login and prepost nodes
+# Uncomment the lines below to locate data on Mistral at DKRZ.
+# Note: Use ``offline: false`` only on login and prepost nodes.
 #auxiliary_data_dir: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/AUX
 #rootpath:
 #  CMIP6: /mnt/lustre02/work/ik1017/CMIP6/data/CMIP6
@@ -158,7 +187,7 @@ drs:
 #  native6: default
 
 # Site-specific entries: ETHZ
-# Uncomment the lines below to locate data at ETHZ
+# Uncomment the lines below to locate data at ETHZ.
 #rootpath:
 #  CMIP6: /net/atmos/data/cmip6
 #  CMIP5: /net/atmos/data/cmip5
@@ -170,7 +199,7 @@ drs:
 #  CMIP3: ETHZ
 
 # Site-specific entries: IPSL
-# Uncomment the lines below to locate data on Ciclad at IPSL
+# Uncomment the lines below to locate data on Ciclad at IPSL.
 #rootpath:
 #  IPSLCM: /
 #  CMIP5: /bdd/CMIP5/output

tests/unit/config/test_config.py

@@ -141,11 +141,11 @@ def test_load_default_config(monkeypatch):
         'config_file': str(default_cfg_file),
         'download_dir': str(Path.home() / 'climate_data'),
         'drs': {
-            'CMIP3': 'default',
-            'CMIP5': 'default',
-            'CMIP6': 'default',
-            'CORDEX': 'default',
-            'obs4MIPs': 'default'
+            'CMIP3': 'ESGF',
+            'CMIP5': 'ESGF',
+            'CMIP6': 'ESGF',
+            'CORDEX': 'ESGF',
+            'obs4MIPs': 'ESGF'
         },
         'exit_on_warning': False,
         'extra_facets_dir': tuple(),
@@ -157,19 +157,7 @@ def test_load_default_config(monkeypatch):
         'remove_preproc_dir': True,
         'resume_from': [],
         'rootpath': {
-            'CMIP3': [str(Path.home() / 'cmip3_inputpath1'),
-                      str(Path.home() / 'cmip3_inputpath2')],
-            'CMIP5': [str(Path.home() / 'cmip5_inputpath1'),
-                      str(Path.home() / 'cmip5_inputpath2')],
-            'CMIP6': [str(Path.home() / 'cmip6_inputpath1'),
-                      str(Path.home() / 'cmip6_inputpath2')],
-            'OBS': [str(Path.home() / 'obs_inputpath')],
-            'OBS6': [str(Path.home() / 'obs6_inputpath')],
-            'obs4MIPs': [str(Path.home() / 'obs4mips_inputpath')],
-            'ana4mips': [str(Path.home() / 'ana4mips_inputpath')],
-            'native6': [str(Path.home() / 'native6_inputpath')],
-            'RAWOBS': [str(Path.home() / 'rawobs_inputpath')],
-            'default': [str(Path.home() / 'default_inputpath')]
+            'default': [str(Path.home() / 'climate_data')]
         },
         'run_diagnostic': True,
         'save_intermediary_cubes': False,