Config file for download cache settings

.travis.yml

@@ -18,10 +18,10 @@ cache:
 matrix:
   fast_finish: true
   include:
-    - env: OGGM_ENV=prepro MPL=
-    - env: OGGM_ENV=models MPL=
-    - env: OGGM_ENV=workflow MPL=--mpl
-    - env: OGGM_ENV=graphics MPL=--mpl
+    - env: OGGM_TEST_ENV=prepro MPL=
+    - env: OGGM_TEST_ENV=models MPL=
+    - env: OGGM_TEST_ENV=workflow MPL=--mpl
+    - env: OGGM_TEST_ENV=graphics MPL=--mpl
 
 before_install:
   - docker pull oggm/untested_base:latest
@@ -40,7 +40,7 @@ install:
     EOF
   - mkdir -p $HOME/dl_cache
   - export OGGM_DOWNLOAD_CACHE=/dl_cache
-  - docker create --name oggm_travis -ti -v $HOME/dl_cache:/dl_cache -e OGGM_DOWNLOAD_CACHE -e OGGM_ENV -e CI -e TRAVIS -e TRAVIS_JOB_ID -e TRAVIS_BRANCH -e TRAVIS_PULL_REQUEST oggm/untested_base:latest /bin/bash /root/oggm/test.sh
+  - docker create --name oggm_travis -ti -v $HOME/dl_cache:/dl_cache -e OGGM_DOWNLOAD_CACHE -e OGGM_TEST_ENV -e CI -e TRAVIS -e TRAVIS_JOB_ID -e TRAVIS_BRANCH -e TRAVIS_PULL_REQUEST oggm/untested_base:latest /bin/bash /root/oggm/test.sh
   - docker cp $PWD oggm_travis:/root/oggm
 script:
   - export OGGM_DOWNLOAD_CACHE=/dl_cache

oggm/cfg.py

@@ -29,6 +29,8 @@
 CACHE_DIR = os.path.join(os.path.expanduser('~'), '.oggm')
 if not os.path.exists(CACHE_DIR):
     os.makedirs(CACHE_DIR)
+# Path to the config file
+CONFIG_FILE = os.path.join(os.path.expanduser('~'), '.oggm_config')
 
 
 class DocumentedDict(dict):
@@ -227,40 +229,18 @@ def initialize(file=None):
     try:
         cp = ConfigObj(file, file_error=True)
     except (ConfigObjError, IOError) as e:
-        log.critical('Config file could not be parsed (%s): %s', file, e)
+        log.critical('Param file could not be parsed (%s): %s', file, e)
         sys.exit()
 
-    homedir = os.path.expanduser('~')
-
-    # Some defaults
-    if cp['working_dir'] == '~':
-        cp['working_dir'] = os.path.join(homedir, 'OGGM_wd')
-    if cp['topo_dir'] == '~':
-        cp['topo_dir'] = os.path.join(homedir, 'OGGM_data', 'topo')
-    if cp['cru_dir'] == '~':
-        cp['cru_dir'] = os.path.join(homedir, 'OGGM_data', 'cru')
-    if cp['rgi_dir'] == '~':
-        cp['rgi_dir'] = os.path.join(homedir, 'OGGM_data', 'rgi')
-
-    # Setup Download-Cache-Dir
-    if os.environ.get('OGGM_DOWNLOAD_CACHE_RO') is not None:
-        cp['dl_cache_readonly'] = bool(strtobool(os.environ.get('OGGM_DOWNLOAD_CACHE_RO')))
-    if os.environ.get('OGGM_DOWNLOAD_CACHE') is not None:
-        cp['dl_cache_dir'] = os.environ.get('OGGM_DOWNLOAD_CACHE')
-
-    PATHS['dl_cache_dir'] = cp['dl_cache_dir']
-    PARAMS['dl_cache_readonly'] = cp.as_bool('dl_cache_readonly')
-
-    if PATHS['dl_cache_dir'] and not os.path.exists(PATHS['dl_cache_dir']):
-        if not PARAMS['dl_cache_readonly']:
-            os.makedirs(PATHS['dl_cache_dir'])
-
     CONTINUE_ON_ERROR = cp.as_bool('continue_on_error')
 
+    # Paths
+    oggm_static_paths()
     PATHS['working_dir'] = cp['working_dir']
-    PATHS['topo_dir'] = cp['topo_dir']
-    PATHS['cru_dir'] = cp['cru_dir']
-    PATHS['rgi_dir'] = cp['rgi_dir']
+    # Default
+    if not PATHS['working_dir']:
+        PATHS['working_dir'] = os.path.join(os.path.expanduser('~'),
+                                            'OGGM_WORKING_DIRECTORY')
     PATHS['dem_file'] = cp['dem_file']
     PATHS['climate_file'] = cp['climate_file']
     PATHS['wgms_rgi_links'] = cp['wgms_rgi_links']
@@ -307,8 +287,8 @@ def initialize(file=None):
     PARAMS[_k] = cp.as_bool(_k)
 
     # Make sure we have a proper cache dir
-    from oggm.utils import _download_oggm_files
-    _download_oggm_files()
+    from oggm.utils import download_oggm_files
+    download_oggm_files()
 
     # Parse RGI metadata
     _d = os.path.join(CACHE_DIR, 'oggm-sample-data-master', 'rgi_meta')
@@ -327,10 +307,9 @@ def initialize(file=None):
            'optimize_inversion_params', 'use_multiple_flowlines',
            'leclercq_rgi_links', 'optimize_thick', 'mpi_recv_buf_size',
            'tstar_search_window', 'use_bias_for_run', 'run_period',
-           'prcp_scaling_factor', 'use_intersects',
-           'dl_cache_dir', 'dl_cache_readonly']
+           'prcp_scaling_factor', 'use_intersects']
     for k in ltr:
-        del cp[k]
+        cp.pop(k, None)
 
     # Other params are floats
     for k in cp:
@@ -343,14 +322,67 @@ def initialize(file=None):
     IS_INITIALIZED = True
 
 
+def oggm_static_paths():
+    """Initialise the OGGM paths from the config file."""
+
+    global PATHS, PARAMS
+
+    # See if the file is there, if not create it
+    if not os.path.exists(CONFIG_FILE):
+        dldir = os.path.join(os.path.expanduser('~'), 'OGGM_DOWNLOADS')
+        config = ConfigObj()
+        config['dl_cache_dir'] = dldir
+        config['tmp_dir'] = ''
+        config['topo_dir'] = ''
+        config['cru_dir'] = ''
+        config['rgi_dir'] = ''
+        config['has_internet'] = True
+        config.filename = CONFIG_FILE
+        config.write()
+
+    # OK, read in the file
+    try:
+        config = ConfigObj(CONFIG_FILE, file_error=True)
+    except (ConfigObjError, IOError) as e:
+        log.critical('Config file could not be parsed (%s): %s',
+                     CONFIG_FILE, e)
+        sys.exit()
+
+    # Check that all keys are here
+    for k in ['dl_cache_dir', 'tmp_dir', 'topo_dir',
+              'cru_dir', 'rgi_dir', 'has_internet']:
+        if k not in config:
+            raise RuntimeError('The oggm config file ({}) should have an '
+                               'entry for {}.'.format(CONFIG_FILE, k))
+
+    # Override defaults with env variables if available
+    if os.environ.get('OGGM_DOWNLOAD_CACHE') is not None:
+        config['dl_cache_dir'] = os.environ.get('OGGM_DOWNLOAD_CACHE')
+
+    if not config['dl_cache_dir']:
+        raise RuntimeError('At the very least, the "dl_cache_dir" entry '
+                           'should be provided in the oggm config file '
+                           '({})'.format(CONFIG_FILE, k))
+
+    # Fill the PATH dict
+    for k, v in config.iteritems():
+        if not v and '_dir' in k:
+            # defaults to the cache dir
+            v = os.path.join(config['dl_cache_dir'], k.replace('_dir', ''))
+        PATHS[k] = os.path.abspath(os.path.expanduser(v))
+
+    # Other
+    PARAMS['has_internet'] = config.as_bool('has_internet')
+
+
 def get_lru_handler(tmpdir=None, maxsize=100, ending='.tif'):
     """LRU handler for a given temporary directory (singleton).
 
     Parameters
     ----------
     tmpdir : str
-        path to the temporary directory to handle. Default is "tmp" in the
-        working directory.
+        path to the temporary directory to handle. Default is 
+        ``cfg.PATHS['tmp_dir']``.
     maxsize : int
         the max number of files to keep in the directory
     ending : str
@@ -360,7 +392,7 @@ def get_lru_handler(tmpdir=None, maxsize=100, ending='.tif'):
 
     # see if we're set up
     if tmpdir is None:
-        tmpdir = os.path.join(PATHS['working_dir'], 'tmp')
+        tmpdir = PATHS['tmp_dir']
     if not os.path.exists(tmpdir):
         os.makedirs(tmpdir)
 

oggm/params.cfg

@@ -3,31 +3,7 @@
 ### Input/Output paths. Set to ~ to default to home directory
 
 # Where OGGM will write its output
-working_dir = ~
-
-# Input directory for topography data. It can be emtpy: in that case OGGM will
-# download the data from the SRTM (open) and ASTER (not open) databases
-topo_dir = ~
-
-# Input directory for CRU TS data. It can be emtpy: in that case OGGM will
-# download the data from https://crudata.uea.ac.uk
-cru_dir = ~
-
-# Input directory for RGI data. It can be emtpy: in that case OGGM will
-# download the data
-rgi_dir = ~
-
-# Cache directory for downloads. If set, all downloaded data will be stored here
-# initially. If a file requested for download is present in the cache, it will be
-# copied from there instead of downloaded again.
-# Can be overridden via OGGM_DOWNLOAD_CACHE environment variable.
-dl_cache_dir =
-
-# Set to True if the download cache is readonly.
-# Downloads will be written to their normal location if neccessary.
-# Alternatively set the OGGM_DOWNLOAD_CACHE_RO environment variable to indicate
-# a read-only cache dir.
-dl_cache_readonly = False
+working_dir =
 
 # Users can specify their own topography file if they want to. In this case,
 # the topo_dir above will be ignored. This is useful for testing, or if you
@@ -36,7 +12,7 @@ dl_cache_readonly = False
 dem_file =
 
 # Users can specify their own climate dataset if they want to. In this case,
-# the cru_dir above will be ignored. This is useful for testing, or if you
+# the static cru_dir will be ignored. This is useful for testing, or if you
 # are simulating a single region with better data.
 # The format of the file is not (yet) very flexible. See the HISTALP data
 # in the sample-data folder for an example:

oggm/tests/__init__.py

@@ -65,7 +65,7 @@
     else:
         # distribute the tests
         RUN_SLOW_TESTS = True
-        env = os.environ.get('OGGM_ENV')
+        env = os.environ.get('OGGM_TEST_ENV')
         if env == 'prepro':
             RUN_PREPRO_TESTS = True
             RUN_MODEL_TESTS = False

oggm/tests/test_prepro.py

@@ -596,10 +596,10 @@ def test_distribute_climate_cru(self):
         climate.process_histalp_nonparallel([gdirs[0]])
         cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
         cru_dir = os.path.dirname(cru_dir)
-        cfg.PATHS['climate_file'] = '~'
+        cfg.PATHS['climate_file'] = ''
         cfg.PATHS['cru_dir'] = cru_dir
         climate.process_cru_data(gdirs[1])
-        cfg.PATHS['cru_dir'] = '~'
+        cfg.PATHS['cru_dir'] = ''
         cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
 
         ci = gdir.read_pickle('climate_info')
@@ -1434,6 +1434,7 @@ def to_optimize(x):
     def test_continue_on_error(self):
 
         cfg.CONTINUE_ON_ERROR = True
+        cfg.PATHS['working_dir'] = self.testdir
 
         hef_file = get_demo_file('Hintereisferner.shp')
         entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]

oggm/tests/test_utils.py

@@ -140,10 +140,8 @@ def setUp(self):
         self.homedir = os.path.expanduser('~')
 
     def test_defaults(self):
-        expected = os.path.join(self.homedir, 'OGGM_wd')
+        expected = os.path.join(self.homedir, 'OGGM_WORKING_DIRECTORY')
         self.assertEqual(cfg.PATHS['working_dir'], expected)
-        expected = os.path.join(self.homedir, 'OGGM_data', 'topo')
-        self.assertEqual(cfg.PATHS['topo_dir'], expected)
 
     def test_pathsetter(self):
         cfg.PATHS['working_dir'] = os.path.join('~', 'my_OGGM_wd')
@@ -154,8 +152,10 @@ def test_pathsetter(self):
 class TestDataFiles(unittest.TestCase):
 
     def setUp(self):
+        cfg.initialize()
         cfg.PATHS['topo_dir'] = TEST_DIR
         cfg.PATHS['working_dir'] = TEST_DIR
+        cfg.PATHS['tmp_dir'] = os.path.join(TEST_DIR, 'extract')
         self.reset_dir()
 
     def tearDown(self):
@@ -179,9 +179,6 @@ def test_download_demo_files(self):
         # Data files
         cfg.initialize()
 
-        lf, df = utils.get_leclercq_files()
-        self.assertTrue(os.path.exists(lf))
-
         lf, df = utils.get_wgms_files()
         self.assertTrue(os.path.exists(lf))
 
@@ -267,6 +264,11 @@ def test_dem3_viewpano_zone(self):
         self.assertTrue(len(z) == 1)
         self.assertEqual('Q01', z[0])
 
+        # normal tile
+        z = utils.dem3_viewpano_zone([107, 107], [69, 69])
+        self.assertTrue(len(z) == 1)
+        self.assertEqual('R48', z[0])
+
         # Alps
         ref = sorted(['K31', 'K32', 'K33', 'L31', 'L32',
                       'L33', 'M31', 'M32', 'M33'])
@@ -342,7 +344,7 @@ def test_srtmdownloadfails(self):
 
         # this zone does not exist
         zone = '41_20'
-        self.assertRaises(FileNotFoundError, utils._download_srtm_file, zone)
+        self.assertTrue(utils._download_srtm_file(zone) is None)
 
     @is_download
     def test_asterdownload(self):
@@ -356,7 +358,8 @@ def test_asterdownload(self):
     @is_download
     def test_gimp(self):
         fp, z = utils.get_topo_file([], [], rgi_region=5)
-        self.assertTrue(os.path.exists(fp))
+        self.assertTrue(os.path.exists(fp[0]))
+        self.assertEqual(z, 'GIMP')
 
     @is_download
     def test_iceland(self):
@@ -366,7 +369,7 @@ def test_iceland(self):
     @is_download
     def test_asterdownloadfails(self):
 
-        # this zone does exist and file should be small enough for download
+        # this zone does not exist
         zone = 'bli'
         unit = 'S75E135'
         self.assertTrue(utils._download_aster_file(zone, unit) is None)
@@ -381,8 +384,6 @@ def test_alternatedownload(self):
     @is_download
     def test_download_cru(self):
 
-        cfg.initialize()
-
         tmp = cfg.PATHS['cru_dir']
         cfg.PATHS['cru_dir'] = TEST_DIR
 
@@ -394,8 +395,6 @@ def test_download_cru(self):
     @is_download
     def test_download_rgi(self):
 
-        cfg.initialize()
-
         tmp = cfg.PATHS['rgi_dir']
         cfg.PATHS['rgi_dir'] = TEST_DIR
 
@@ -415,3 +414,26 @@ def test_download_dem3_viewpano(self):
         zone = 'U44'
         fp = utils._download_dem3_viewpano(zone)
         self.assertTrue(os.path.exists(fp))
+
+    @is_download
+    def test_download_dem3_viewpano_fails(self):
+
+        # this zone does not exist
+        zone = 'dummy'
+        fp = utils._download_dem3_viewpano(zone)
+        self.assertTrue(fp is None)
+
+    @is_download
+    def test_auto_topo(self):
+        # Test for combine
+        fdem, src = utils.get_topo_file([6, 14], [41, 41])
+        self.assertEqual(src, 'SRTM')
+        self.assertEqual(len(fdem), 2)
+        for fp in fdem:
+            self.assertTrue(os.path.exists(fp))
+
+        fdem, src = utils.get_topo_file([-143, -131], [61, 61])
+        self.assertEqual(src, 'DEM3')
+        self.assertEqual(len(fdem), 3)
+        for fp in fdem:
+            self.assertTrue(os.path.exists(fp))

oggm/tests/test_workflow.py

@@ -116,7 +116,7 @@ def up_to_inversion(reset=False):
         # Use histalp for the actual inversion test
         cfg.PARAMS['temp_use_local_gradient'] = True
         cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oetztal.nc')
-        cfg.PATHS['cru_dir'] = '~'
+        cfg.PATHS['cru_dir'] = ''
         workflow.climate_tasks(gdirs)
         with open(CLI_LOGF, 'wb') as f:
             pickle.dump('histalp', f)
@@ -147,7 +147,7 @@ def up_to_distrib(reset=False):
         # Use CRU
         cfg.PARAMS['prcp_scaling_factor'] = 2.5
         cfg.PARAMS['temp_use_local_gradient'] = False
-        cfg.PATHS['climate_file'] = '~'
+        cfg.PATHS['climate_file'] = ''
         cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
         cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
         with warnings.catch_warnings():