Standardise how paths are handled (#189)

* ads_utils: update module to pathlib

and do some reformatting.

On branch maf88/154-standardize-paths-management
Changes to be committed:
	modified:   AxonDeepSeg/ads_utils.py
	modified:   test/test_ads_utils.py

* Revert styling changes

* Revert more styling changes

* ads_utils: Refactor to use tempfile and pathlib

* test_dataset_building.py:  Refactor to use pathlib

* Refactor to pathlib and remove unused imports

* test_launch_morph[...]:  Refactor to use pathlib

* test_launch_perf[...]: Refactor to use pathlib

* test_noise_simulation: Refactor to use pathlib

* test_segmentation_scoring: Refactor to use pathlib

* statistics_generation and test_st[...] use pathlib

* test_generate_axons_from_myelin: refact to pathlib

* test_get_masks: use pathib

* test_merge_masks: use pathlib

* test_visualize.py : use pathlib

* test_ads_utils.py: use pathlib

* Fix typo

* test_config_tools.py: use pathlib

* FIX convert Path to str for specific functions

* Fix  missing () in test_config_tools.py

* test_segment.py: Update to pathlib

* test_train_network.py: Refactor to use pathlib

* setup.py: Refactor path handling to pathlib

* config_tools.py: use pathlib

* integrity_test.py: use pathlib

* input_data.py: use pathlib

* integrity_test: FIX wrong syntax for path

* input_data.py: FIX dataset size eval and docstring

* conf.py: use pathlib

* merge_masks.py: use pathlib

* get_masks.py: Refactor to handle paths correctly

* submission_generator_tools.py:  refactor 2 pathlib

* train_network.py: Refactor to pathlib

* test_train_network: FIX Path to str conversion

* train_network: FIX  tf.train.Saver.save() param type

* segment.py : update to pathlib

* apply_model.py: use pathlib

* apply_model.py: FIX bad indent

* dataset_building.py: Refactor to pathlib

* mapping_results.py: use pathlib

* get_masks.py: use pathlib

* compute_morphometrics: Refactor to pathlib

* launch_morphometrics_computation.py: use pathlib

* launch_performance_m[...]: remove unused lib/var

* trainingforhelios.py: use pathlib

* generate_axons_from_myelin.py: use pathlib

* guide_local.py:  clean and refactor to pathlib

* config_to_csv.py: update to pathlib

* visualize.py:  update to pathlib

* config_to_csv: FIX indent and trailing whitespaces

* FIX path handled as path not strings

* test_visualize.py: FIX unwanted str() conversion

* launch_perf[...]: Fix unecessary #TODO

and move the issue to a Github issue

* dataset_building.py: Remove os.walk to use pathlib

* test_compute_morph[...]: FIX format

* test_compute_mor[-]: Fix wrong call to os.path

* test_compute_mor[-]:  Fix typo

* dataset_building.py: Revert Var Names

* get_masks.py: Fix Confusing Var Names

* visualize.py: Fix Conflicting Var Name

* confing_to_csv.py: Fix Conflicting Var Name

* Update Documentation For V2.2

* segment.py: Add  Conversion To pathlib.Path

* segment.py.:  Fix Missng Args Case

*  Update tests to assume string paths at function call; add  str to Path conversions at start of funcs (#207)

* Revert tested API to use string path

* Add convert_path function & tests

* WIP: implemented path conversion and applied it.

* Remove test files

* Resolve convert_path bug with lists

* Add doc for convert_path()

AxonDeepSeg/ads_utils.py

@@ -1,15 +1,21 @@
+"""
+AxonDeepSeg utilities module.
+"""
+
 import os
 import sys
+from pathlib import Path
 import configparser
 from distutils.util import strtobool
-import raven
-from tqdm import tqdm
+import cgi
 import tempfile
 import zipfile
 import requests
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util import Retry
-import cgi
+from tqdm import tqdm
+import raven
+
 
 DEFAULT_CONFIGFILE = "axondeepseg.cfg"
 
@@ -70,7 +76,7 @@ def _main_thread_terminated(self):
 def config_setup():
 
     config_path = get_config_path()
-    
+
     if 'pytest' in sys.modules:
         bugTracking = bool(0)
     else:
@@ -89,7 +95,7 @@ def config_setup():
 
     config = configparser.ConfigParser()
     config['Global'] = {
-        'bugTracking': bugTracking,
+        'bugTracking': bugTracking
     }
 
     with open(config_path, 'w') as configFile:
@@ -101,10 +107,8 @@ def get_config_path():
     """Get the full path of the AxonDeepSeg configuration file.
     :return: String with the full path to the ADS config file.
     """
-    return os.path.join(
-                    os.path.expanduser("~"),
-                    DEFAULT_CONFIGFILE
-                    )
+    return Path.home() / DEFAULT_CONFIGFILE
+
 
 def read_config():
     """Read the system configuration file.
@@ -113,11 +117,11 @@ def read_config():
 
     config_path = get_config_path()
 
-    if not os.path.exists(config_path):
+    if not config_path.exists():
         raise IOError("Could not find configuration file.")
 
     config = configparser.ConfigParser()
-    config.read(config_path)
+    config.read(str(config_path))
 
     return config
 
@@ -129,7 +133,7 @@ def init_ads():
 
     config_path = get_config_path()
 
-    if not os.path.isfile(config_path):
+    if not config_path.is_file():
         config_setup()
     else:
         pass
@@ -153,7 +157,7 @@ def init_error_client(bugTracking):
                         processors=(
                             'raven.processors.RemoveStackLocalsProcessor',
                             'raven.processors.SanitizePasswordsProcessor')
-                            )
+                        )
 
             traceback_to_server(client)
 
@@ -177,7 +181,6 @@ def download_data(url_data):
     """ Downloads and extracts zip files from the web.
     :return: 0 - Success, 1 - Encountered an exception.
     """
-
     # Download
     try:
         print('Trying URL: %s' % url_data)
@@ -192,31 +195,60 @@ def download_data(url_data):
         else:
             print("Unexpected: link doesn't provide a filename")
 
-        tmp_path = os.path.join(tempfile.mkdtemp(), zip_filename)
-
-        with open(tmp_path, 'wb') as tmp_file:
-            total = int(response.headers.get('content-length', 1))
-            tqdm_bar = tqdm(total=total, unit='B', unit_scale=True, desc="Downloading", ascii=True)
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:
-                    tmp_file.write(chunk)
-                    dl_chunk = len(chunk)
-                    tqdm_bar.update(dl_chunk)
-            tqdm_bar.close()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_path = Path(tmpdir) / zip_filename
+            with open(tmp_path, 'wb') as tmp_file:
+                total = int(response.headers.get('content-length', 1))
+                tqdm_bar = tqdm(total=total, unit='B', unit_scale=True, desc="Downloading", ascii=True)
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        tmp_file.write(chunk)
+                        dl_chunk = len(chunk)
+                        tqdm_bar.update(dl_chunk)
+                tqdm_bar.close()
+            # Unzip
+            print("Unzip...")
+            try:
+                zf = zipfile.ZipFile(str(tmp_path))
+                zf.extractall(".")
+            except (zipfile.BadZipfile):
+                print('ERROR: ZIP package corrupted. Please try downloading again.')
+                return 1
+            print("--> Folder created: " + str(Path.cwd() / Path(zip_filename).stem))
     except Exception as e:
         print("ERROR: %s" % e)
         return 1
-
-    # Unzip
-    print("Unzip...")
-    try:
-        zf = zipfile.ZipFile(tmp_path)
-        zf.extractall(".")
-    except (zipfile.BadZipfile):
-        print('ERROR: ZIP package corrupted. Please try downloading again.')
-        return 1
-    print("--> Folder created: " + os.path.join(os.path.abspath(os.curdir), zip_filename.strip(".zip")))
     return 0
 
+def convert_path(object_path):
+    """ Convert path
+    Convert path or list of paths to Path() objects.
+    If None type, returns None.
+
+    :param object_path: string, Path() object, None, or a list of these.
+    :return: Path() object, None, or a list of these.
+    """
+    if isinstance(object_path, list):
+        path_list = []
+        for path_iter in object_path:
+            if isinstance(path_iter, Path):
+                path_list.append(path_iter)
+            elif isinstance(path_iter, str):
+                path_list.append(Path(path_iter))
+            elif path_iter == None:
+                path_list.append(None)
+            else:
+                raise TypeError('Paths, folder names, and filenames must be either strings or pathlib.Path objects. object_path was type: ' + str(type(object_path)))
+        return path_list
+    else:
+        if isinstance(object_path, Path):
+            return object_path
+        elif isinstance(object_path, str):
+            return Path(object_path)
+        elif object_path == None:
+            return None
+        else:
+            raise TypeError('Paths, folder names, and filenames must be either strings or pathlib.Path objects. object_path was type: ' + str(type(object_path)))
+
 # Call init_ads() automatically when module is imported
 init_ads()