Merge pull request #285 from intake/fix_config_path

fix config paths

intake/config.py

@@ -77,27 +77,43 @@ def load_conf(fn=None):
 
 
 def intake_path_dirs(path):
-    """Return a list of directories from the intake path."""
-    separator = ';' if os.name == 'nt' else ':'
-    return path.split(separator)
-
-
-reset_conf()
-load_conf(cfile())
-
-# environment variables take precedence over conf file
-for key, envvar in [['cache_dir', 'INTAKE_CACHE_DIR'],
-                    ['catalog_path', 'INTAKE_PATH'],
-                    ['persist_path', 'INTAKE_PERSIST_PATH']]:
-    if envvar in os.environ:
-        conf[key] = make_path_posix(os.environ[envvar])
-for key, envvar in [['cache_disabled', 'INTAKE_DISABLE_CACHING'],
-                   ['cache_download_progress', 'INTAKE_CACHE_PROGRESS']]:
-    if envvar in os.environ:
-        conf[key] = os.environ[envvar].lower() in ['true', 't', 'y', 'yes']
-if 'INTAKE_LOG_LEVEL' in os.environ:
-    conf['logging'] = os.environ['INTAKE_LOG_LEVEL']
+    """Return a list of directories from the intake path.
 
+    If a string, perhaps taken from an environment variable, then the
+    list of paths will be split on the character ":" for posix of ";" for
+    windows. Protocol indicators ("protocol://") will be ignored.
+    """
+    if isinstance(path, (list, tuple)):
+        return path
+    import re
+    pattern = re.compile(";" if os.name == 'nt' else r"(?<!:):(?![:/])")
+    return pattern.split(path)
+
+
+def load_env():
+    """Analyse enviroment variables and update conf accordingly"""
+    # environment variables take precedence over conf file
+    for key, envvar in [['cache_dir', 'INTAKE_CACHE_DIR'],
+                        ['catalog_path', 'INTAKE_PATH'],
+                        ['persist_path', 'INTAKE_PERSIST_PATH']]:
+        if envvar in os.environ:
+            conf[key] = make_path_posix(os.environ[envvar])
+    conf['catalog_path'] = intake_path_dirs(conf['catalog_path'])
+    for key, envvar in [['cache_disabled', 'INTAKE_DISABLE_CACHING'],
+                        ['cache_download_progress', 'INTAKE_CACHE_PROGRESS']]:
+        if envvar in os.environ:
+            conf[key] = os.environ[envvar].lower() in ['true', 't', 'y', 'yes']
+    if 'INTAKE_LOG_LEVEL' in os.environ:
+        conf['logging'] = os.environ['INTAKE_LOG_LEVEL']
+
+
+def reload_all():
+    reset_conf()
+    load_conf(cfile())
+    load_env()
+
+
+reload_all()
 
 logger.setLevel(conf['logging'])
 ch = logging.StreamHandler()

intake/tests/test_config.py

@@ -21,7 +21,7 @@
 ])
 def test_load_conf(conf):
     # This test will only work if your config is set to default
-    inconf = config.conf.copy()
+    inconf = config.defaults.copy()
     expected = inconf.copy()
     with temp_conf(conf) as fn:
         config.load_conf(fn)
@@ -41,6 +41,18 @@ def test_basic():
         with server(env=env, wait=5000):
             r = requests.get('http://localhost:5000/v1/info')
             assert r.ok
+    with temp_conf({}) as fn:
+        env = os.environ.copy()
+        env['INTAKE_CONF'] = os.path.dirname(fn)
+        with server(env=env, wait=5000):
+            r = requests.get('http://localhost:5000/v1/info')
+            assert r.ok
+    with temp_conf({}) as fn:
+        env = os.environ.copy()
+        env['INTAKE_CONF'] = os.path.dirname(fn) + ":/nonexistent"
+        with server(env=env, wait=5000):
+            r = requests.get('http://localhost:5000/v1/info')
+            assert r.ok
 
 
 def test_cli():
@@ -83,3 +95,13 @@ def test_conf_auth():
                                  storage_options={'headers':
                                                   {'intake-secret': 'test'}})
             assert 'entry1' in cat
+
+
+@pytest.mark.skipif(os.name == 'nt', reason="Paths are different on win")
+def test_pathdirs():
+    assert config.intake_path_dirs([]) == []
+    assert config.intake_path_dirs(['paths']) == ['paths']
+    assert config.intake_path_dirs("") == [""]
+    assert config.intake_path_dirs("path1:path2") == ['path1', 'path2']
+    assert config.intake_path_dirs("s3://path1:s3://path2") == [
+        's3://path1', 's3://path2']