init_glacier_directories to replace init_glacier_regions (#983)

* revert old init_glacier_regions behavior * WIP: init_glacier_directories * More changes * fix tests * switch to new func everywhere * continue check on docs * more docs and whats new * last doc tweaks
OGGM · Mar 25, 2020 · 291c8fc · 291c8fc
1 parent 1f14413
commit 291c8fc
Show file tree

Hide file tree

Showing 30 changed files with 572 additions and 355 deletions.
diff --git a/benchmarks/track_model_results.py b/benchmarks/track_model_results.py
@@ -36,7 +36,7 @@ def setup_cache(self):
 
         gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
 
-        tasks.define_glacier_region(gdir, entity=entity)
+        tasks.define_glacier_region(gdir)
         tasks.glacier_masks(gdir)
         tasks.compute_centerlines(gdir)
         tasks.initialize_flowlines(gdir)
@@ -172,10 +172,11 @@ def setup_cache(self):
         rgidf = rgidf.sort_values('Area', ascending=False)
 
         # Go - initialize glacier directories
-        gdirs = workflow.init_glacier_regions(rgidf)
+        gdirs = workflow.init_glacier_directories(rgidf)
 
         # Preprocessing tasks
         task_list = [
+            tasks.define_glacier_region,
             tasks.glacier_masks,
             tasks.compute_centerlines,
             tasks.initialize_flowlines,
@@ -276,7 +277,7 @@ def setup_cache(self):
         entity = gpd.read_file(get_demo_file('01_rgi60_Columbia.shp')).iloc[0]
         gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
 
-        tasks.define_glacier_region(gdir, entity=entity)
+        tasks.define_glacier_region(gdir)
         tasks.glacier_masks(gdir)
         tasks.compute_centerlines(gdir)
         tasks.initialize_flowlines(gdir)

diff --git a/docs/_code/prepare_climate.py b/docs/_code/prepare_climate.py
@@ -22,7 +22,7 @@
 entity = gpd.read_file(get_demo_file('HEF_MajDivide.shp')).iloc[0]
 gdir = oggm.GlacierDirectory(entity, base_dir=base_dir, reset=True)
 
-tasks.define_glacier_region(gdir, entity=entity)
+tasks.define_glacier_region(gdir)
 tasks.glacier_masks(gdir)
 tasks.compute_centerlines(gdir)
 tasks.initialize_flowlines(gdir)

diff --git a/docs/_code/prepare_flowlines.py b/docs/_code/prepare_flowlines.py
@@ -12,7 +12,7 @@
 entity = gpd.read_file(get_demo_file('HEF_MajDivide.shp')).iloc[0]
 gdir = oggm.GlacierDirectory(entity, base_dir=base_dir, reset=True)
 
-tasks.define_glacier_region(gdir, entity=entity)
+tasks.define_glacier_region(gdir)
 tasks.glacier_masks(gdir)
 tasks.compute_centerlines(gdir)
 tasks.initialize_flowlines(gdir)

diff --git a/docs/api.rst b/docs/api.rst
@@ -25,6 +25,7 @@ Tools to set-up and run OGGM.
     cfg.set_logging_config
     cfg.set_intersects_db
     cfg.reset_working_dir
+    workflow.init_glacier_directories
     workflow.init_glacier_regions
     workflow.execute_entity_task
     workflow.gis_prepro_tasks
@@ -174,9 +175,10 @@ pre-processed state available on the OGGM servers:
     cfg.initialize()  # always initialize before an OGGM task
     # The working directory is where OGGM will store the run's data
     cfg.PATHS['working_dir'] = os.path.join(gettempdir(), 'Docs_GlacierDir')
-    gdirs = workflow.init_glacier_regions('RGI60-11.00897', from_prepro_level=1,
-                                          prepro_border=10)
-    gdir = gdirs[0]  # init_glacier_regions always returns a list
+    gdirs = workflow.init_glacier_directories('RGI60-11.00897',
+                                              from_prepro_level=1,
+                                              prepro_border=10)
+    gdir = gdirs[0]  # init_glacier_directories always returns a list
 
 
 We just downloaded the minimal input for a glacier directory. The
@@ -353,9 +355,10 @@ including the model flowlines. This is achieved by choosing preprocessing level
     cfg.initialize()  # always initialize before an OGGM task
     # The working directory is where OGGM will store the run's data
     cfg.PATHS['working_dir'] = os.path.join(gettempdir(), 'Docs_GlacierDir2')
-    gdirs = workflow.init_glacier_regions('RGI60-11.00897', from_prepro_level=4,
-                                          prepro_border=10)
-    gdir = gdirs[0]  # init_glacier_regions always returns a list
+    gdirs = workflow.init_glacier_directories('RGI60-11.00897',
+                                              from_prepro_level=4,
+                                              prepro_border=10)
+    gdir = gdirs[0]  # init_glacier_directories always returns a list
 
     fls = gdir.read_pickle('model_flowlines')
     fls

diff --git a/docs/input-data.rst b/docs/input-data.rst
@@ -127,7 +127,7 @@ of the current stable OGGM version. If you want to change these parameters,
 you'll have to do a full run from scratch using the :ref:`rawdata`.
 
 To start from a pre-processed state, simply use the
-:py:func:`workflow.init_glacier_regions` function with the
+:py:func:`workflow.init_glacier_directories` function with the
 ``from_prepro_level`` and ``prepro_border`` keyword arguments set to the
 values of your choice.
 
@@ -178,9 +178,9 @@ Here is an example with the Hintereisferner in the Alps:
 
     f, axs = plt.subplots(2, 2, figsize=(8, 6))
     for ax, border in zip(np.array(axs).flatten(), [10, 80, 160, 250]):
-        gdir = workflow.init_glacier_regions('RGI60-11.00897',
-                                             from_prepro_level=1,
-                                             prepro_border=border)
+        gdir = workflow.init_glacier_directories('RGI60-11.00897',
+                                                 from_prepro_level=1,
+                                                 prepro_border=border)
         graphics.plot_domain(gdir, ax=ax, title='Border: {}'.format(border),
                              add_colorbar=False,
                              lonlat_contours_kwargs={'add_tick_labels':False})
@@ -290,7 +290,7 @@ EndDate             not included
 For Greenland and Antarctica, OGGM does not take into account the
 connectivity level between the Glaciers and the Ice sheets.
 We recommend to the users to think about this before they
-run the task: ``workflow.init_glacier_regions()``.
+run the task: ``workflow.init_glacier_directories``.
 
 .. _Randolph Glacier Inventory (RGI): https://www.glims.org/RGI/
 
@@ -490,8 +490,9 @@ have access to the timeseries through the glacier directory:
 
 .. ipython:: python
 
-    gdir = workflow.init_glacier_regions('RGI60-11.00897', from_prepro_level=4,
-                                         prepro_border=10)[0]
+    gdir = workflow.init_glacier_directories('RGI60-11.00897',
+                                             from_prepro_level=4,
+                                             prepro_border=10)[0]
     mb = gdir.get_ref_mb_data()
     @savefig plot_ref_mbdata.png width=100%
     mb[['ANNUAL_BALANCE']].plot(title='WGMS data: Hintereisferner')

diff --git a/docs/practicalities.rst b/docs/practicalities.rst
@@ -250,7 +250,7 @@ you have to deal with:
   :py:func:`utils.gdir_to_tar` and :py:func:`utils.base_dir_to_tar` functions
   to create compressed, aggregated files of your directories. You can
   later initialize new directories from these tar files with the `from_tar`
-  keyword argument in :py:func:`workflow.init_glacier_regions`.
+  keyword argument in :py:func:`workflow.init_glacier_directories`.
 
 
 Run per RGI region, not globally

diff --git a/docs/run_examples/_code/mb_crossval.py b/docs/run_examples/_code/mb_crossval.py
@@ -36,7 +36,7 @@
 rgidf = gpd.read_file(os.path.join(WORKING_DIR, 'mb_ref_glaciers.shp'))
 
 # Go - initialize glacier directories
-gdirs = workflow.init_glacier_regions(rgidf)
+gdirs = workflow.init_glacier_directories(rgidf)
 
 # Cross-validation
 file = os.path.join(cfg.PATHS['working_dir'], 'ref_tstars.csv')

diff --git a/docs/run_examples/_code/run_errors.py b/docs/run_examples/_code/run_errors.py
@@ -36,7 +36,7 @@
 log.workflow('Number of glaciers: {}'.format(len(rgi_ids)))
 
 # Go - get the pre-processed glacier directories
-gdirs = workflow.init_glacier_regions(rgi_ids, from_prepro_level=4)
+gdirs = workflow.init_glacier_directories(rgi_ids, from_prepro_level=4)
 
 # We can step directly to the experiment!
 # Random climate representative for the recent climate (1985-2015)

diff --git a/docs/run_examples/_code/run_inversion.py b/docs/run_examples/_code/run_inversion.py
@@ -40,8 +40,9 @@
 
 # Go - get the pre-processed glacier directories
 # We start at level 3, because we need all data for the inversion
-gdirs = workflow.init_glacier_regions(rgidf, from_prepro_level=3,
-                                      prepro_border=10)
+gdirs = workflow.init_glacier_directories(rgidf,
+                                          from_prepro_level=3,
+                                          prepro_border=10)
 
 # Default parameters
 # Deformation: from Cuffey and Patterson 2010

diff --git a/docs/run_examples/_code/run_reference_mb_glaciers.py b/docs/run_examples/_code/run_reference_mb_glaciers.py
@@ -70,7 +70,7 @@
 
 # We have to check which of them actually have enough mb data.
 # Let OGGM do it:
-gdirs = workflow.init_glacier_regions(rgidf)
+gdirs = workflow.init_glacier_directories(rgidf)
 
 # We need to know which period we have data for
 log.info('Process the climate data...')
@@ -96,10 +96,11 @@
 rgidf = rgidf.sort_values('Area', ascending=False)
 
 # Go - initialize glacier directories
-gdirs = workflow.init_glacier_regions(rgidf)
+gdirs = workflow.init_glacier_directories(rgidf)
 
 # Prepro tasks
 task_list = [
+    tasks.define_glacier_region,
     tasks.glacier_masks,
     tasks.compute_centerlines,
     tasks.initialize_flowlines,

diff --git a/docs/run_examples/_code/run_rgi_region.py b/docs/run_examples/_code/run_rgi_region.py
@@ -52,7 +52,7 @@
 log.workflow('Number of glaciers: {}'.format(len(rgidf)))
 
 # Go - get the pre-processed glacier directories
-gdirs = workflow.init_glacier_regions(rgidf, from_prepro_level=4)
+gdirs = workflow.init_glacier_directories(rgidf, from_prepro_level=4)
 
 # We can step directly to a new experiment!
 # Random climate representative for the recent climate (1985-2015)

diff --git a/docs/run_examples/_code/run_with_spinup.py b/docs/run_examples/_code/run_with_spinup.py
@@ -33,7 +33,8 @@
 cfg.PARAMS['border'] = 80
 
 # Go - initialize glacier directories
-gdirs = workflow.init_glacier_regions(['RGI60-11.00897'], from_prepro_level=4)
+gdirs = workflow.init_glacier_directories(['RGI60-11.00897'],
+                                          from_prepro_level=4)
 
 # Additional climate file (CESM)
 cfg.PATHS['cesm_temp_file'] = get_demo_file('cesm.TREFHT.160001-200512'

diff --git a/docs/run_examples/run_errors.rst b/docs/run_examples/run_errors.rst
@@ -25,17 +25,20 @@ applications.
 
 If everything went "well", you should see an output similar to::
 
-    2019-02-21 15:51:39: oggm.cfg: Using configuration file: /home/mowglie/Documents/git/oggm-fork/oggm/params.cfg
-    2019-02-21 15:51:39: __main__: Starting OGGM run
-    2019-02-21 15:51:39: __main__: Number of glaciers: 3
-    2019-02-21 15:51:39: oggm.workflow: init_glacier_regions from prepro level 4 on 3 glaciers.
-    2019-02-21 15:51:39: oggm.workflow: Execute entity task gdir_from_prepro on 3 glaciers
-    2019-02-21 15:51:39: oggm.workflow: Multiprocessing: using all available processors (N=8)
-    2019-02-21 15:51:40: oggm.workflow: Execute entity task run_random_climate on 3 glaciers
-    2019-02-21 15:51:40: oggm.core.flowline: RuntimeError occurred during task run_random_climate on RGI60-11.03295: Need a valid `model_flowlines` file. If you explicitly want to use `inversion_flowlines`, set use_inversion_flowlines=True.
-    2019-02-21 15:52:11: oggm.core.flowline: RuntimeError occurred during task run_random_climate on RGI60-11.00897: Glacier exceeds domain boundaries.
-    2019-02-21 15:52:11: oggm.workflow: Execute entity task glacier_statistics on 3 glaciers
-    2019-02-21 15:52:11: __main__: OGGM is done! Time needed: 0:00:32
+    2020-03-24 18:54:32: oggm.cfg: Using configuration file: /home/mowglie/disk/Dropbox/HomeDocs/git/oggm-fork/oggm/params.cfg
+    2020-03-24 18:54:32: oggm.cfg: Multiprocessing switched ON according to the parameter file.
+    2020-03-24 18:54:32: oggm.cfg: Multiprocessing: using all available processors (N=8)
+    2020-03-24 18:54:32: __main__: Starting OGGM run
+    2020-03-24 18:54:32: __main__: Number of glaciers: 3
+    2020-03-24 18:54:32: oggm.workflow: init_glacier_directories from prepro level 4 on 3 glaciers.
+    2020-03-24 18:54:32: oggm.workflow: Execute entity task gdir_from_prepro on 3 glaciers
+    2020-03-24 18:54:32: oggm.workflow: Initializing multiprocessing pool with N=8 processes.
+    2020-03-24 18:55:00: oggm.workflow: Execute entity task run_random_climate on 3 glaciers
+    2020-03-24 18:55:00: oggm.core.flowline: RuntimeError occurred during task run_random_climate on RGI60-11.03295: Need a valid `model_flowlines` file. If you explicitly want to use `inversion_flowlines`, set use_inversion_flowlines=True.
+    2020-03-24 18:55:10: oggm.core.flowline: RuntimeError occurred during task run_random_climate on RGI60-11.00897: Glacier exceeds domain boundaries, at year: 135.83333333333334
+    2020-03-24 18:55:31: oggm.workflow: Execute entity task glacier_statistics on 3 glaciers
+    2020-03-24 18:55:31: __main__: OGGM is done! Time needed: 0:00:59
+
 
 Dealing with errors
 -------------------

diff --git a/docs/run_examples/run_inversion.rst b/docs/run_examples/run_inversion.rst
@@ -31,18 +31,18 @@ Here, we run the inversion algorithm for all glaciers in the Pyrenees.
 
 If everything went well, you should see an output similar to::
 
-    2019-02-16 22:16:19: oggm.cfg: Using configuration file: /home/mowglie/Documents/git/oggm-fork/oggm/params.cfg
-    2019-02-16 22:16:20: __main__: Starting OGGM inversion run
-    2019-02-16 22:16:20: __main__: Number of glaciers: 35
-    2019-02-16 22:16:20: oggm.workflow: init_glacier_regions from prepro level 3 on 35 glaciers.
-    2019-02-16 22:16:20: oggm.workflow: Execute entity task gdir_from_prepro on 35 glaciers
-    2019-02-16 22:16:20: oggm.workflow: Multiprocessing: using all available processors (N=8)
-    2019-02-16 22:16:21: oggm.workflow: Execute entity task mass_conservation_inversion on 35 glaciers
-    2019-02-16 22:16:21: oggm.workflow: Execute entity task filter_inversion_output on 35 glaciers
+    2020-03-25 12:31:52: oggm.cfg: Using configuration file: /home/mowglie/disk/Dropbox/HomeDocs/git/oggm-fork/oggm/params.cfg
+    2020-03-25 12:31:52: oggm.cfg: Multiprocessing switched ON according to the parameter file.
+    2020-03-25 12:31:52: oggm.cfg: Multiprocessing: using all available processors (N=8)
+    2020-03-25 12:31:53: __main__: Starting OGGM inversion run
+    2020-03-25 12:31:53: __main__: Number of glaciers: 35
+    2020-03-25 12:31:53: oggm.workflow: init_glacier_directories from prepro level 3 on 35 glaciers.
+    2020-03-25 12:31:55: oggm.workflow: Execute entity task gdir_from_prepro on 35 glaciers
+    2020-03-25 12:32:00: oggm.workflow: Execute entity task mass_conservation_inversion on 35 glaciers
     (...)
-    2019-02-16 22:16:25: oggm.workflow: Execute entity task filter_inversion_output on 35 glaciers
-    2019-02-16 22:16:25: oggm.workflow: Execute entity task glacier_statistics on 35 glaciers
-    2019-02-16 22:16:25: __main__: OGGM is done! Time needed: 0:00:05
+    2020-03-25 12:32:33: oggm.workflow: Execute entity task filter_inversion_output on 35 glaciers
+    2020-03-25 12:32:33: oggm.workflow: Execute entity task glacier_statistics on 35 glaciers
+    2020-03-25 12:32:34: __main__: OGGM is done! Time needed: 0:00:41
 
 
 Analysing the output

diff --git a/docs/run_examples/run_rgi_region.rst b/docs/run_examples/run_rgi_region.rst
@@ -29,16 +29,19 @@ Script
 If everything went well, you should see an output similar to::
 
 
-    2019-02-16 17:50:51: oggm.cfg: Using configuration file: /home/mowglie/Documents/git/oggm-fork/oggm/params.cfg
-    2019-02-16 17:50:52: __main__: Starting OGGM run
-    2019-02-16 17:50:52: __main__: Number of glaciers: 54
-    2019-02-16 17:50:52: oggm.workflow: init_glacier_regions from prepro level 4 on 54 glaciers.
-    2019-02-16 17:50:52: oggm.workflow: Execute entity task gdir_from_prepro on 54 glaciers
-    2019-02-16 17:50:52: oggm.workflow: Multiprocessing: using all available processors (N=8)
-    2019-02-16 17:50:54: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
-    2019-02-16 17:51:44: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
-    2019-02-16 17:52:36: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
-    2019-02-16 17:54:11: __main__: OGGM is done! Time needed: 0:03:20
+    2020-03-25 12:34:22: oggm.cfg: Using configuration file: /home/mowglie/disk/Dropbox/HomeDocs/git/oggm-fork/oggm/params.cfg
+    2020-03-25 12:34:22: oggm.cfg: Multiprocessing switched ON according to the parameter file.
+    2020-03-25 12:34:22: oggm.cfg: Multiprocessing: using all available processors (N=8)
+    2020-03-25 12:34:23: __main__: Starting OGGM run
+    2020-03-25 12:34:23: __main__: Number of glaciers: 54
+    2020-03-25 12:34:23: oggm.workflow: init_glacier_directories from prepro level 4 on 54 glaciers.
+    2020-03-25 12:34:25: oggm.workflow: Execute entity task gdir_from_prepro on 54 glaciers
+    2020-03-25 12:34:38: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
+    2020-03-25 12:35:22: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
+    2020-03-25 12:36:12: oggm.workflow: Execute entity task run_random_climate on 54 glaciers
+    2020-03-25 12:38:03: oggm.workflow: Execute entity task glacier_statistics on 54 glaciers
+    2020-03-25 12:38:04: __main__: OGGM is done! Time needed: 0:03:42
+
 
 Some analyses
 -------------

diff --git a/docs/whats-new.rst b/docs/whats-new.rst
@@ -19,6 +19,15 @@ Breaking changes
   temperature standard deviation (:pull:`978`). The previous default was
   wrong and should not be the default.
   By `Fabien Maussion <https://github.com/fmaussion>`_
+- Added a new "glacier directory initialization" global task:
+  `init_glacier_directories` (:pull:`983`, :issue:`965`). It replaces
+  `init_glacier_regions` and covers all its functionality, except that
+  it does *not* process the DEM data (this was a confusing "feature" of
+  `init_glacier_regions`). The old task `init_glacier_regions` is officially
+  deprecated but without warnings for now. Since it is a very widely used
+  task, we prefer to deprecate it in a slow cycle: first, change the
+  documentation, deprecate later.
+  By `Fabien Maussion <https://github.com/fmaussion>`_
 
 Enhancements
 ~~~~~~~~~~~~

diff --git a/oggm/cfg.py b/oggm/cfg.py
@@ -633,8 +633,8 @@ def set_intersects_db(path_or_gdf=None):
     """Set the glacier intersection database for OGGM to use.
 
     It is now set automatically by the
-    :func:`oggm.workflow.init_glacier_regions` task, but setting it manually
-    can be useful for a slightly faster run initialization.
+    :func:`oggm.workflow.init_glacier_directories` task, but setting it
+    manually can be useful for a slightly faster run initialization.
 
     See :func:`oggm.utils.get_rgi_intersects_region_file` for how to obtain
     such data.

diff --git a/oggm/cli/benchmark.py b/oggm/cli/benchmark.py
@@ -127,8 +127,8 @@ def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
 
     # Initialize working directories
     start = time.time()
-    gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
-    _add_time_to_df(odf, 'init_glacier_regions', time.time()-start)
+    gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
+    _add_time_to_df(odf, 'init_glacier_directories', time.time()-start)
 
     # Pre-download other files just in case
     if test_crudir is None:
@@ -139,6 +139,7 @@ def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
 
     # Tasks
     task_list = [
+        tasks.define_glacier_region,
         tasks.process_cru_data,
         tasks.glacier_masks,
         tasks.compute_centerlines,

diff --git a/oggm/cli/prepro_levels.py b/oggm/cli/prepro_levels.py
@@ -247,7 +247,8 @@ def _time_log():
         rgidf['DEM_SOURCE'] = dem_source.upper()
 
     # L1 - go
-    gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
+    gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
+    workflow.execute_entity_task(tasks.define_glacier_region, gdirs)
 
     # Glacier stats
     sum_dir = os.path.join(base_dir, 'L1', 'summary')