Merge pull request #30 from neuro-ml/dev

Add s3 and redis configs

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
 
     steps:
       - uses: actions/checkout@v3

bev/__version__.py

@@ -1 +1 @@
-__version__ = '0.9.1'
+__version__ = '0.10.0'

bev/config/location.py

@@ -1,15 +1,17 @@
+import os
 import warnings
 from pathlib import Path
 from typing import Optional, Sequence
 
+import boto3
 import yaml
 from jboc import collect
 from paramiko.config import SSHConfig
-from tarn import SCP, DiskDict, Fanout, Level, Levels, Location, Nginx
+from tarn import S3, SCP, DiskDict, Fanout, Level, Levels, Location, Nginx, RedisLocation
 from tarn.config import CONFIG_NAME as STORAGE_CONFIG_NAME, StorageConfig as TarnStorageConfig
 from tarn.utils import mkdir
 
-from .compat import field_validator, NoExtra, core_schema, model_validate, model_dump
+from .compat import NoExtra, core_schema, field_validator, model_dump, model_validate
 from .registry import RegistryError, add_type, find, register
 
 
@@ -187,3 +189,43 @@ def from_special(x, passthrough: bool = True):
 
     if passthrough:
         return x
+
+
+@register('s3')
+class S3Config(LocationConfig):
+    url: str
+    bucket: str
+    credentials_file: Optional[str] = None
+
+    def build(self) -> Optional[Location]:
+        # we carefully patch the os.environ here
+        # FIXME: this is not thread safe though
+        if self.credentials_file is not None:
+            path = Path(self.credentials_file).expanduser()
+            name = 'AWS_SHARED_CREDENTIALS_FILE'
+
+            current_path = os.environ.get(name)
+            if current_path and Path(current_path).expanduser() != path:
+                raise ValueError(
+                    'Cannot overwrite the current value in "AWS_SHARED_CREDENTIALS_FILE": '
+                    f'{current_path} vs {path}'
+                )
+
+            os.environ[name] = str(path)
+
+        s3 = boto3.client('s3', endpoint_url=self.url)
+        return S3(s3, self.bucket)
+
+
+@register('redis')
+class RedisConfig(LocationConfig):
+    url: str
+    prefix: str = '_'
+
+    @classmethod
+    def from_special(cls, v):
+        if isinstance(v, str):
+            return cls(url=v)
+
+    def build(self) -> Optional[Location]:
+        return RedisLocation(self.url, self.prefix)

pyproject.toml

@@ -3,17 +3,17 @@ name = 'bev'
 dynamic = ['version', 'dependencies']
 description = 'A small manager for versioned data'
 readme = 'README.md'
-requires-python = '>=3.6'
+requires-python = '>=3.7'
 license = { file = 'LICENSE' }
 keywords = ['data', 'version control']
 authors = [
-    { name = 'NeuroML Group', email = 'maxs987@gmail.com' }
+    { name = 'Max', email = 'max@ira-labs.com' },
+    { name = 'Talgat', email = 't.saparov@ira-labs.com' },
 ]
 classifiers = [
     'Development Status :: 5 - Production/Stable',
     'License :: OSI Approved :: Apache Software License',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
@@ -39,9 +39,7 @@ build-backend = 'setuptools.build_meta'
 include = ['bev']
 
 [tool.setuptools.package-data]
-bev = [
-    'py.typed',
-]
+bev = ['py.typed']
 
 [tool.setuptools.dynamic]
 version = { attr = 'bev.__version__.__version__' }
@@ -58,7 +56,7 @@ ignore = ['W503', 'E203', 'B028', 'C408']
 per-file-ignores = [
     '**/__init__.py:F401,F403',
     'tests/*:I251',
-    'bev/cli/*:B008'
+    'bev/cli/*:B008',
 ]
 max-line-length = 120
 banned-modules = 'bev.* = Use relative imports'

setup.py

@@ -1,12 +1,11 @@
 from pathlib import Path
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 classifiers = [
     'Development Status :: 5 - Production/Stable',
     'License :: OSI Approved :: Apache Software License',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
@@ -39,7 +38,7 @@
     keywords=['data', 'version control'],
     classifiers=classifiers,
     install_requires=requirements,
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     entry_points={
         'console_scripts': [
             'bev = bev.cli.entrypoint:entrypoint',