Use different baselines images depending on freetype version

appveyor.yml

@@ -13,7 +13,7 @@ notifications:
 environment:
   matrix:
     - PYTHON: "C:\\Python3-conda64"
-      PYTHON_VERSION: "3.5"
+      PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
 
 install:
@@ -22,6 +22,7 @@ install:
 
   # Create environment with desired python version and activate it
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET GDAL_DATA=%PYTHON%\\Library\\share\\gdal"
   - "conda install --yes --quiet python=%PYTHON_VERSION%"
 
   # Check that we have the expected version and architecture for Python
@@ -30,7 +31,7 @@ install:
 
   # install dependencies
   - "python -m pip install --upgrade pip"
-  - "conda install --yes --quiet -c conda-forge geopandas matplotlib Pillow joblib netCDF4 scikit-image configobj pytest pyproj numpy rasterio krb5 xarray boto3"
+  - "conda install --yes --quiet -c conda-forge gdal libgdal geopandas matplotlib Pillow joblib netCDF4 scikit-image configobj pytest pyproj numpy rasterio krb5 xarray boto3"
   - "pip install motionless"
   - "pip install git+https://github.com/fmaussion/salem.git"
   - "pip install -e ."

oggm/tests/__init__.py

@@ -7,6 +7,7 @@
 import unittest
 import logging
 import matplotlib
+import matplotlib.ft2font
 import pandas as pd
 import geopandas as gpd
 import numpy as np
@@ -31,7 +32,12 @@
 if LooseVersion(matplotlib.__version__) >= LooseVersion('2'):
     HAS_MPL_FOR_TESTS = True
     BASELINE_DIR = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data-master',
-                                'baseline_images', '2.0.x')
+                                'baseline_images')
+    ftver = LooseVersion(matplotlib.ft2font.__freetype_version__)
+    if ftver >= LooseVersion('2.7.0'):
+        BASELINE_DIR = os.path.join(BASELINE_DIR, 'alt')
+    else:
+        BASELINE_DIR = os.path.join(BASELINE_DIR, '2.0.x')
 
 
 # Some control on which tests to run (useful to avoid too long tests)

oggm/utils.py

@@ -93,7 +93,7 @@ def _get_download_lock():
     return lock
 
 
-def _cached_download_helper(cache_obj_name, dl_func):
+def _cached_download_helper(cache_obj_name, dl_func, reset=False):
     """Helper function for downloads.
 
     Takes care of checking if the file is already cached.
@@ -107,14 +107,14 @@ def _cached_download_helper(cache_obj_name, dl_func):
         cache_dir = fb_cache_dir
         cache_ro = False
 
-    cache_path = os.path.join(cache_dir, cache_obj_name)
-    if os.path.isfile(cache_path):
-        return cache_path
-
     fb_path = os.path.join(fb_cache_dir, cache_obj_name)
-    if os.path.isfile(fb_path):
+    if not reset and os.path.isfile(fb_path):
         return fb_path
 
+    cache_path = os.path.join(cache_dir, cache_obj_name)
+    if not reset and os.path.isfile(cache_path):
+        return cache_path
+
     if cache_ro:
         cache_path = fb_path
 
@@ -130,7 +130,7 @@ def _cached_download_helper(cache_obj_name, dl_func):
     return cache_path
 
 
-def _urlretrieve(url, cache_obj_name=None, *args, **kwargs):
+def _urlretrieve(url, cache_obj_name=None, reset=False, *args, **kwargs):
     """Wrapper around urlretrieve, to implement our caching logic.
 
     Instead of accepting a destination path, it decided where to store the file
@@ -146,10 +146,10 @@ def _dlf(cache_path):
         urlretrieve(url, cache_path, *args, **kwargs)
         return cache_path
 
-    return _cached_download_helper(cache_obj_name, _dlf)
+    return _cached_download_helper(cache_obj_name, _dlf, reset)
 
 
-def _progress_urlretrieve(url, cache_name=None):
+def _progress_urlretrieve(url, cache_name=None, reset=False):
     """Downloads a file, returns its local path, and shows a progressbar."""
 
     try:
@@ -165,22 +165,22 @@ def _upd(count, size, total):
                     pbar[0].start(UnknownLength)
             pbar[0].update(min(count * size, total))
             sys.stdout.flush()
-        res = _urlretrieve(url, cache_obj_name=cache_name, reporthook=_upd)
+        res = _urlretrieve(url, cache_obj_name=cache_name, reset=reset, reporthook=_upd)
         try:
             pbar[0].finish()
         except:
             pass
         return res
     except ImportError:
-        return _urlretrieve(url, cache_obj_name=cache_name)
+        return _urlretrieve(url, cache_obj_name=cache_name, reset=reset)
 
 
-def aws_file_download(aws_path):
+def aws_file_download(aws_path, cache_name=None, reset=False):
     with _get_download_lock():
-        return _aws_file_download_unlocked(aws_path)
+        return _aws_file_download_unlocked(aws_path, cache_name, reset)
 
 
-def _aws_file_download_unlocked(aws_path, cache_name=None):
+def _aws_file_download_unlocked(aws_path, cache_name=None, reset=False):
     """Download a file from the AWS drive s3://astgtmv2/
 
     **Note:** you need AWS credentials for this to work.
@@ -212,10 +212,10 @@ def _dlf(cache_path):
                 raise
         return cache_path
 
-    return _cached_download_helper(cache_obj_name, _dlf)
+    return _cached_download_helper(cache_obj_name, _dlf, reset)
 
 
-def file_downloader(www_path, retry_max=5, cache_name=None):
+def file_downloader(www_path, retry_max=5, cache_name=None, reset=False):
     """A slightly better downloader: it tries more than once."""
 
     local_path = None
@@ -224,7 +224,7 @@ def file_downloader(www_path, retry_max=5, cache_name=None):
         # Try to download
         try:
             retry_counter += 1
-            local_path = _progress_urlretrieve(www_path, cache_name=cache_name)
+            local_path = _progress_urlretrieve(www_path, cache_name=cache_name, reset=reset)
             # if no error, exit
             break
         except HTTPError as err:

oggm/workflow.py

@@ -42,9 +42,14 @@ def init_mp_pool(reset=False):
     global_lock = mp.Manager().Lock()
     mpp = cfg.PARAMS['mp_processes']
     if mpp == -1:
-        mpp = mp.cpu_count()
-        log.info('Multiprocessing: using all available '
-                 'processors (N={})'.format(mp.cpu_count()))
+        try:
+            mpp = int(os.environ['SLURM_JOB_CPUS_PER_NODE'])
+            log.info('Multiprocessing: using slurm allocated '
+                     'processors (N={})'.format(mpp))
+        except KeyError:
+            mpp = mp.cpu_count()
+            log.info('Multiprocessing: using all available '
+                     'processors (N={})'.format(mpp))
     else:
         log.info('Multiprocessing: using the requested number of '
                  'processors (N={})'.format(mpp))