Add configuration module (#23)

MANIFEST.in

@@ -0,0 +1,8 @@
+include LICENSE
+include README.md
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+
+recursive-include funnel *.py
+recursive-include funnel *.yaml

funnel/__init__.py

@@ -4,6 +4,7 @@
 from pkg_resources import DistributionNotFound, get_distribution
 
 from .cache import CacheStore
+from .config import config
 from .metadata_db.main import BaseMetadataStore, MemoryMetadataStore, SQLMetadataStore
 from .registry import registry
 from .serializers import pick_serializer

funnel/config.py

@@ -0,0 +1,47 @@
+import pathlib
+import typing
+
+import dask
+import pkg_resources
+import pydantic
+import yaml
+
+from .metadata_db.main import MemoryMetadataStore, SQLMetadataStore
+from .registry import registry
+
+default_config_dir = pathlib.Path('~').expanduser() / '.config' / 'funnel'
+default_config_dir.mkdir(parents=True, exist_ok=True)
+
+config_file_path = pathlib.Path(pkg_resources.resource_filename('funnel', 'funnel.yaml'))
+default_config_path = default_config_dir / 'funnel.yaml'
+
+if not default_config_path.exists():
+    default_config_path.write_text(config_file_path.read_text())
+
+
+class Config(pydantic.BaseSettings):
+    """
+    Configuration settings for the Funnel.
+    """
+
+    metadata_store: typing.Union[SQLMetadataStore, MemoryMetadataStore] = None
+
+    class Config:
+        validate_assignment = True
+
+    def load_config(self, config_path: str):
+        """
+        Loads the config from a file.
+        """
+        with open(config_path) as f:
+            config = dask.config.expand_environment_variables(yaml.safe_load(f))
+        metadata_store_type = config['metadata_store']['type']
+        del config['metadata_store']['type']
+        self.metadata_store = registry.metadata_store.get(metadata_store_type)(
+            cache_store_options=config['cache_store'],
+            metadata_store_options=config['metadata_store'],
+        )
+
+
+config = Config()
+config.load_config(default_config_path)

funnel/funnel.yaml

@@ -0,0 +1,10 @@
+cache_store:
+  path: $TMPDIR/funnel/cache/
+  readonly: false
+  on_duplicate_key: skip
+  storage_options:
+
+metadata_store:
+  type: sql # memory or sql
+  readonly: false
+  database_url: sqlite:///$HOME/.config/funnel/funnel.db

funnel/metadata_db/main.py

@@ -8,7 +8,7 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy_utils import create_database, database_exists
 
-from ..cache import CacheStore, DuplicateKeyEnum
+from ..cache import CacheStore
 from ..registry import registry
 from . import models, schemas
 
@@ -168,37 +168,12 @@ def df(self):
 
 
 @registry.metadata_store('memory')
-def memory_metadata_store(
-    cache_store: str,
-    *,
-    cache_store_readonly: bool = False,
-    storage_options: dict = None,
-    on_duplicate_key: DuplicateKeyEnum = 'skip',
-    database_readonly: bool = False,
-):
-    cache_store = CacheStore(
-        cache_store,
-        readonly=cache_store_readonly,
-        on_duplicate_key=on_duplicate_key,
-        storage_options=storage_options,
-    )
-    return MemoryMetadataStore(cache_store, readonly=database_readonly)
+def memory_metadata_store(cache_store_options: dict, metadata_store_options: dict):
+    cache_store = CacheStore(**cache_store_options)
+    return MemoryMetadataStore(cache_store, **metadata_store_options)
 
 
 @registry.metadata_store('sql')
-def sql_metadata_store(
-    cache_store: str,
-    *,
-    cache_store_readonly: bool = False,
-    storage_options: dict = None,
-    on_duplicate_key: DuplicateKeyEnum = 'skip',
-    database_readonly: bool = False,
-    database_url: str = None,
-):
-    cache_store = CacheStore(
-        cache_store,
-        readonly=cache_store_readonly,
-        on_duplicate_key=on_duplicate_key,
-        storage_options=storage_options,
-    )
-    return SQLMetadataStore(cache_store, database_url=database_url, readonly=database_readonly)
+def sql_metadata_store(cache_store_options: dict, metadata_store_options: dict):
+    cache_store = CacheStore(**cache_store_options)
+    return SQLMetadataStore(cache_store, **metadata_store_options)

funnel/metadata_db/schemas.py

@@ -18,10 +18,7 @@ class ArtifactBase(pydantic.BaseModel):
     created_at: datetime.datetime = datetime.datetime.utcnow()
 
 
-class ArtifactCreate(ArtifactBase):
-    ...
-
-
 class Artifact(ArtifactBase):
     class Config:
         orm_mode = True
+        validate_assignment = True

setup.cfg

@@ -7,7 +7,7 @@ select = B,C,E,F,W,T4,B9
 
 [isort]
 known_first_party=funnel
-known_third_party=catalogue,fsspec,pandas,pkg_resources,pydantic,pytest,setuptools,sqlalchemy,sqlalchemy_utils,xarray
+known_third_party=catalogue,dask,fsspec,pandas,pkg_resources,pydantic,pytest,setuptools,sqlalchemy,sqlalchemy_utils,xarray,yaml
 multi_line_output=3
 include_trailing_comma=True
 force_grid_wrap=0

tests/test_config.py

@@ -0,0 +1,13 @@
+import pydantic
+import pytest
+
+import funnel
+
+
+def test_validate_error():
+    with pytest.raises(pydantic.ValidationError):
+        funnel.config.metadata_store = 5
+
+
+def test_default_config():
+    assert isinstance(funnel.config, pydantic.BaseSettings)