Merge pull request #29 from dephell/dotenv

Dotenv

dephell/actions/__init__.py

@@ -4,6 +4,7 @@
 # app
 from ._autocomplete import make_bash_autocomplete, make_zsh_autocomplete
 from ._converting import attach_deps
+from ._dotenv import read_dotenv
 from ._downloads import get_downloads_by_category, get_total_downloads
 from ._editorconfig import make_editorconfig
 from ._entrypoints import get_entrypoints
@@ -36,11 +37,12 @@
     'get_venv',
     'get_version_from_file',
     'get_version_from_project',
-    'git_tag',
     'git_commit',
+    'git_tag',
     'make_bash_autocomplete',
     'make_editorconfig',
     'make_json',
     'make_zsh_autocomplete',
+    'read_dotenv',
     'roman2arabic',
 ]

dephell/actions/_dotenv.py

@@ -0,0 +1,43 @@
+import shlex
+from codecs import decode
+from pathlib import Path
+from string import Template
+from typing import Dict
+
+
+def read_dotenv(path: Path, env_vars: Dict[str, str] = None) -> Dict[str, str]:
+    if env_vars is None:
+        env_vars = dict()
+    else:
+        env_vars = env_vars.copy()
+
+    if path.is_dir():
+        path = path / '.env'
+    if not path.exists():
+        return env_vars
+
+    with path.open('r', encoding='utf-8') as stream:
+        for line in stream:
+            line = line.strip()
+            if not line or line[0] == '#':
+                continue
+            key, value = line.split('=', 1)
+
+            # clean key
+            key = key.strip()
+            if key.startswith('export '):
+                key = key.replace('export ', '', 1)
+                key = key.strip()
+            if key[0] == '$':
+                key = key[1:]
+
+            # clean and substitute value
+            value = ' '.join(shlex.split(value, comments=True))
+            value = decode(value, 'unicode-escape')
+            if '$' in value:
+                value = value.replace(r'\$', '$$')  # escaping
+                value = Template(value).safe_substitute(env_vars)
+
+            env_vars[key] = value
+
+    return env_vars

dephell/commands/jail_try.py

@@ -11,7 +11,7 @@
 from dephell_venvs import VEnv
 
 # app
-from ..context_tools import env_var
+from ..context_tools import override_env_vars
 from ..actions import get_python, get_resolver
 from ..config import builders
 from ..controllers import analize_conflict
@@ -95,7 +95,7 @@ def __call__(self) -> bool:
 
             # run
             self.logger.info('running...')
-            with env_var(key='PYTHONSTARTUP', value=str(startup_path)):
+            with override_env_vars({'PYTHONSTARTUP': str(startup_path)}):
                 result = subprocess.run([str(executable)] + command[1:])
             if result.returncode != 0:
                 self.logger.error('command failed', extra=dict(code=result.returncode))

dephell/commands/venv_run.py

@@ -1,15 +1,17 @@
 # built-in
 import shlex
 import subprocess
+import os
 from argparse import REMAINDER, ArgumentParser
 from pathlib import Path
 
 # project
 from dephell_venvs import VEnvs
 
 # app
-from ..actions import get_python, get_resolver
+from ..actions import get_python, get_resolver, read_dotenv
 from ..config import builders
+from ..context_tools import override_env_vars
 from ..controllers import analize_conflict
 from ..models import Requirement
 from ..package_manager import PackageManager
@@ -36,6 +38,7 @@ def get_parser(cls) -> ArgumentParser:
         return parser
 
     def __call__(self) -> bool:
+        # get command
         command = self.args.name
         if not command:
             command = self.config.get('command')
@@ -45,6 +48,7 @@ def __call__(self) -> bool:
         if isinstance(command, str):
             command = shlex.split(command)
 
+        # get and make venv
         venvs = VEnvs(path=self.config['venv'])
         venv = venvs.get(Path(self.config['project']), env=self.config.env)
         if not venv.exists():
@@ -58,6 +62,7 @@ def __call__(self) -> bool:
             venv.create(python_path=python.path)
             self.logger.info('venv created', extra=dict(path=venv.path))
 
+        # install executable
         executable = venv.bin_path / command[0]
         if not executable.exists():
             self.logger.warning('executable is not found in venv, trying to install...', extra=dict(
@@ -66,13 +71,23 @@ def __call__(self) -> bool:
             result = self._install(name=command[0], python_path=venv.python_path)
             if not result:
                 return False
-
         if not executable.exists():
             self.logger.error('package installed, but executable is not found')
             return False
 
+        # get env vars
+        env_vars = os.environ.copy()
+        if 'vars' in self.config:
+            env_vars.update(self.config['vars'])
+        env_vars = read_dotenv(
+            path=Path(self.config['project']),
+            env_vars=env_vars,
+        )
+
+        # run
         self.logger.info('running...')
-        result = subprocess.run([str(executable)] + command[1:])
+        with override_env_vars(env_vars):
+            result = subprocess.run([str(executable)] + command[1:])
         if result.returncode != 0:
             self.logger.error('command failed', extra=dict(code=result.returncode))
             return False

dephell/commands/venv_shell.py

@@ -1,4 +1,5 @@
 # built-in
+import os
 from argparse import ArgumentParser
 from pathlib import Path
 
@@ -9,8 +10,9 @@
 from dephell_venvs import VEnvs
 
 # app
-from ..actions import get_python
+from ..actions import get_python, read_dotenv
 from ..config import builders
+from ..context_tools import override_env_vars
 from .base import BaseCommand
 
 
@@ -34,6 +36,7 @@ def get_parser(cls) -> ArgumentParser:
         return parser
 
     def __call__(self) -> bool:
+        # get and create venv
         venvs = VEnvs(path=self.config['venv'])
         venv = venvs.get(Path(self.config['project']), env=self.config.env)
         if not venv.exists():
@@ -42,6 +45,17 @@ def __call__(self) -> bool:
             self.logger.debug('choosen python', extra=dict(version=python.version))
             venv.create(python_path=python.path)
 
+        # get env vars
+        env_vars = os.environ.copy()
+        if 'vars' in self.config:
+            env_vars.update(self.config['vars'])
+        env_vars = read_dotenv(
+            path=Path(self.config['project']),
+            env_vars=env_vars,
+        )
+
         shells = Shells(bin_path=venv.bin_path)
-        shells.run()
+        with override_env_vars(env_vars):
+            shells.run()
+        self.logger.info('shell closed')
         return True

dephell/config/scheme.py

@@ -54,6 +54,12 @@
     # venv
     'venv':     dict(type='string', required=True),
     'python':   dict(type='string', required=False),
+    'vars': dict(
+        type='dict',
+        keyschema={'type': 'string'},
+        valueschema={'type': 'string'},
+        required=False,
+    ),
 
     # other
     'cache': dict(

dephell/context_tools.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from contextlib import contextmanager
+from typing import Dict
 
 
 @contextmanager
@@ -27,16 +28,14 @@ def nullcontext(value=None):
 
 
 @contextmanager
-def env_var(key, value):
-    old_value = os.environ.get(key)
-    os.environ[key] = value
+def override_env_vars(env_vars: Dict[str, str]):
+    old_vars = os.environ.copy()
+    os.environ.update(env_vars)
     try:
         yield
     finally:
-        if old_value is None:
-            del os.environ[key]
-        else:
-            os.environ[key] = old_value
+        os.environ.clear()
+        os.environ.update(old_vars)
 
 
 @contextmanager

docs/cmd-venv-run.md

@@ -23,3 +23,53 @@ In this case command can be omitted:
 ```bash
 $ dephell venv run --env=docs
 ```
+
+## Environment variables
+
+This command passes next [environment variables](https://en.wikipedia.org/wiki/Environment_variable) into running command:
+
+1. Your current environment variables.
+1. Values from `vars` in config.
+1. Values from [.env](https://github.com/theskumar/python-dotenv) file.
+
+example of `.env` file:
+
+```bash
+export POSTGRES_USERNAME="dephell"
+export POSTGRES_PASSWORD="PasswordExample"
+export POSTGRES_URL="psql://$POSTGRES_USERNAME:$POSTGRES_PASSWORD@localhost"
+```
+
+DepHell supports any format of `.env` file: `export` word optional, quotes optional, `=` can be surrounded by spaces. However, we recommend to use above format, because it allows you to use [source command](https://bash.cyberciti.biz/guide/Source_command) to load these variables in your current shell.
+
+Features for `.env` file:
+
+1. Parameters expansion. In the example above `POSTGRES_URL` value will be expanded into `psql://dephell:PasswordExample@localhost`. If variable does not exist DepHell won't touch it. You can explicitly escape $ sign (`\$`) to avoid expansion.
+1. Escape sequences. You can insert escape sequences like `\n` in values, and DepHell will process it.
+
+Config example:
+
+```toml
+[tool.dephell.main]
+vars = {PYTHONPATH = "."}
+command = "python"
+
+[tool.dephell.flake8]
+vars = {TOXENV = "flake8"}
+command = "tox"
+```
+
+Use `.env` for secret things like database credentials and `vars` in config for some environment-specific settings for running commands like environment for flake.
+
+If you want to pass temporary variable that not intended to be stored in any file then just set this variable in your current shell:
+
+```bash
+$ CHECK=me dephell venv run python -c "print(__import__('os').environ['CHECK'])"
+INFO running...
+me
+INFO command successfully completed
+```
+
+## See also
+
+1. [dephell venv shell](cmd-venv-shell) to activate virtual environment for your current shell.

docs/cmd-venv-shell.md

@@ -16,3 +16,9 @@ Supported shells:
 ```bash
 $ dephell venv shell --env=docs
 ```
+
+This command build environment variables in the same way as [dephell venv run](cmd-venv-run).
+
+## See also
+
+1. [dephell venv run](cmd-venv-run) to run single command in a virtual environment.

tests/test_actions/test_dotenv.py

@@ -0,0 +1,52 @@
+import pytest
+
+from dephell.actions import read_dotenv
+
+
+# https://github.com/jpadilla/django-dotenv/blob/master/tests.py
+# https://github.com/motdotla/dotenv/blob/master/tests/.env
+@pytest.mark.parametrize('lines, expected', [
+    (['a=b # lol'], {'a': 'b'}),
+
+    # strip spaces
+    (['FOO=bar'], {'FOO': 'bar'}),
+    (['FOO =bar'], {'FOO': 'bar'}),
+    (['FOO= bar'], {'FOO': 'bar'}),
+    (['FOO=bar  '], {'FOO': 'bar'}),
+    (['  FOO=bar'], {'FOO': 'bar'}),
+    (['take =   me out  '], {'take': 'me out'}),
+
+    # quotes
+    (['FOO="bar"'], {'FOO': 'bar'}),
+    (["FOO='bar'"], {'FOO': 'bar'}),
+
+    # key formats
+    (['FOO.BAR=foobar'], {'FOO.BAR': 'foobar'}),
+
+    # empty
+    (['FOO='], {'FOO': ''}),
+    (['FOO=   '], {'FOO': ''}),
+
+    # substitution
+    (['FOO=test', 'BAR=$FOO'], {'FOO': 'test', 'BAR': 'test'}),
+    (['FOO=test', 'BAR=${FOO}bar'], {'FOO': 'test', 'BAR': 'testbar'}),
+
+    # escaping
+    ([r'FOO="escaped\"bar"'], {'FOO': 'escaped"bar'}),
+    (['FOO=test', r'BAR="foo\$FOO"'], {'FOO': 'test', 'BAR': 'foo$FOO'}),
+    (['FOO=test', r'BAR="foo\${FOO}"'], {'FOO': 'test', 'BAR': 'foo${FOO}'}),
+
+    # escape sequences
+    ([r'take="me\nout"'], {'take': 'me\nout'}),
+    ([r'take="me\out"'], {'take': r'me\out'}),
+
+    # comments
+    (['take=me # out'], {'take': 'me'}),
+    (['take="me # to" # church'], {'take': 'me # to'}),
+    (['take="me" # to # church'], {'take': 'me'}),
+    (['take="me # out"'], {'take': 'me # out'}),
+    (['# take', 'me=out'], {'me': 'out'}),
+])
+def test_read_dotenv(temp_path, lines, expected):
+    (temp_path / '.env').write_text('\n'.join(lines))
+    assert read_dotenv(temp_path) == expected