From f5d23bf053d9f19e01316e78616dae51f4053d8a Mon Sep 17 00:00:00 2001 From: Ian Ross Date: Wed, 3 May 2023 13:56:26 +0200 Subject: [PATCH] Cleanup --- .github/workflows/ci.yml | 25 +++++ .github/workflows/python-publish.yml | 2 +- CHANGELOG.md | 12 ++- MANIFEST.in | 1 + Makefile | 5 +- pyproject.toml | 3 + requirements.txt | 2 + src/dotenv_vault/__version__.py | 2 +- src/dotenv_vault/main.py | 152 +++++++++++++++++++++++---- src/dotenv_vault/test_vault.py | 60 +++++++++++ src/dotenv_vault/vault.py | 103 ------------------ 11 files changed, 240 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 MANIFEST.in create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 src/dotenv_vault/test_vault.py delete mode 100644 src/dotenv_vault/vault.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b42817c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: Test Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run tests + run: | + make test diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index c5b25eb..a4719c1 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel twine build - name: Build package env: TWINE_USERNAME: __token__ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a2abd1..877c021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -## [Unreleased](https://github.com/dotenv-org/python-dotenv-vault/compare/v0.2.0...master) +## [Unreleased](https://github.com/dotenv-org/python-dotenv-vault/compare/v0.5.0...master) +## 0.5.0 + +### Added + + - Reorganise and simplify code + - Make API correspond more closely to `python-dotenv` + - Improve error handling + - Add tests and CI + - Upgrade to `build` for release build + ## 0.4.1 ### Added diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fc49293 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include CHANGELOG.md diff --git a/Makefile b/Makefile index 52b8b67..8406218 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ clean-pyc: find . -name '*~' -exec rm -f {} + build: clean - python setup.py sdist bdist_wheel + python -m build uninstall_local: pip uninstall python-dotenv-vault -y @@ -21,7 +21,8 @@ uninstall_local: install_local: pip install . -test: uninstall_local build install_local +test: install_local + python -m unittest -v dotenv_vault.test_vault release: build twine check dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4aff98b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b0124ee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +python-dotenv~=0.21.0 +cryptography<41.0.0,>=3.1.0 diff --git a/src/dotenv_vault/__version__.py b/src/dotenv_vault/__version__.py index 4fee6c7..c16cf99 100644 --- a/src/dotenv_vault/__version__.py +++ b/src/dotenv_vault/__version__.py @@ -1,7 +1,7 @@ __title__ = "python-dotenv-vault" __description__ = "Decrypt .env.vault file." __url__ = "https://github.com/dotenv-org/python-dotenv-vault" -__version__ = "0.4.1" +__version__ = "0.5.0" __author__ = "dotenv" __author_email__ = "mot@dotenv.org" __license__ = "MIT" diff --git a/src/dotenv_vault/main.py b/src/dotenv_vault/main.py index dfac139..d9500bf 100644 --- a/src/dotenv_vault/main.py +++ b/src/dotenv_vault/main.py @@ -1,15 +1,14 @@ from __future__ import annotations +from base64 import b64decode +import io import os -import logging -from typing import (IO, Optional,Union) -from dotenv.main import load_dotenv as load_dotenv_file +from typing import (IO, Optional, Union) +from urllib.parse import urlparse, parse_qsl -from .vault import DotEnvVault - -logging.basicConfig(level = logging.INFO) - -logger = logging.getLogger(__name__) +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.exceptions import InvalidTag +import dotenv.main as dotenv def load_dotenv( @@ -20,22 +19,137 @@ def load_dotenv( interpolate: bool = True, encoding: Optional[str] = "utf-8", ) -> bool: + """This function will read your encrypted env file and load it + into the environment for this process. + + Call this function as close as possible to the start of your + program (ideally in main). + + If the `DOTENV_KEY` environment variable is set, `load_dotenv` + will load encrypted environment settings from the `.env.vault` + file in the current path. + + If the `DOTENV_KEY` environment variable is not set, `load_dotenv` + falls back to the behavior of the python-dotenv library, loading a + specified (unencrypted) environment file. + + Other parameters to `load_dotenv` are passed througg to the + python-dotenv loader. In particular, whether `load_dotenv` + overrides existing environment settings or not is determined by + the `override` flag. + """ - parameters are the same as python-dotenv library. - This is to inject the parameters to evironment variables. - """ - dotenv_vault = DotEnvVault() - if dotenv_vault.dotenv_key: - logger.info('Loading env from encrypted .env.vault') - vault_stream = dotenv_vault.parsed_vault(dotenv_path=dotenv_path) - # we're going to override the .vault to any existing keys in local - return load_dotenv_file(stream=vault_stream, override=True) + if "DOTENV_KEY" in os.environ: + vault_stream = parse_vault(open(".env.vault")) + return dotenv.load_dotenv( + dotenv_path=".env.vault", + stream=vault_stream, + verbose=verbose, + override=override, + interpolate=interpolate + ) else: - return load_dotenv_file( + return dotenv.load_dotenv( dotenv_path=dotenv_path, stream=stream, verbose=verbose, override=override, interpolate=interpolate, encoding=encoding - ) + ) + + +class DotEnvVaultError(Exception): + pass + + +KEY_LENGTH = 64 + + +def parse_vault(vault_stream: io.IOBase) -> io.StringIO: + """Parse information from DOTENV_KEY, and decrypt vault. + """ + dotenv_key = os.environ.get("DOTENV_KEY") + if dotenv_key is None: + raise DotEnvVaultError("NOT_FOUND_DOTENV_KEY: Cannot find ENV['DOTENV_KEY']") + + # Use the python-dotenv library to read the .env.vault file. + vault = dotenv.DotEnv(dotenv_path=".env.vault", stream=vault_stream) + + # Extract segments from the DOTENV_KEY environment variable one by + # one and retrieve the corresponding ciphertext from the vault + # data. + keys = [] + for dotenv_key_entry in [i.strip() for i in dotenv_key.split(',')]: + key, environment_key = parse_key(dotenv_key_entry) + + ciphertext = vault.dict().get(environment_key) + + if not ciphertext: + raise DotEnvVaultError(f"NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment {environment_key} in your .env.vault file. Run 'npx dotenv-vault build' to include it.") + + keys.append({ + 'encrypted_key': key, + 'ciphertext': ciphertext + }) + + # Try decrypting environments one-by-one in the order they appear + # in the DOTENV_KEY environment variable. + decrypted = _key_rotation(keys=keys) + + # Return the decrypted data as a text stream that we can pass to + # the python-dotenv library. + return io.StringIO(decrypted.decode('utf-8')) + + +def parse_key(dotenv_key): + # Parse a single segment of the DOTENV_KEY environment variable. + # These segments are in the form of URIs (see + # https://www.dotenv.org/docs/security/dotenv-key). + uri = urlparse(dotenv_key) + + # The 64-character encryption key is stored in the password field + # of the URI, possibly with a prefix. + key = uri.password + if len(key) < KEY_LENGTH: + raise DotEnvVault('INVALID_DOTENV_KEY: Key part must be 64 characters long (or more)') + + # The environment is provided in the URI's query parameters. + params = dict(parse_qsl(uri.query)) + vault_environment = params.get('environment') + if not vault_environment: + raise DotEnvVaultError('INVALID_DOTENV_KEY: Missing environment part') + + # Form the key used to store the ciphertext for this environment's + # settings in the .env.vault file. + environment_key = f'DOTENV_VAULT_{vault_environment.upper()}' + + return key, environment_key + + +def _decrypt(ciphertext: str, key: str) -> bytes: + """decrypt method will decrypt via AES-GCM + return: decrypted keys in bytes + """ + # Remove any prefix from the encryption key (at this point, we + # know that the key is at least 64 characters in length) and set + # up the AES cipher. + aesgcm = AESGCM(bytes.fromhex(key[-KEY_LENGTH:])) + + # Decrypt the ciphertext: this is base64-encoded in the .env.vault + # file, and the first 12 bytes of the decoded data are used as the + # AES nonce value. + ciphertext = b64decode(ciphertext) + return aesgcm.decrypt(ciphertext[:12], ciphertext[12:], b'') + + +def _key_rotation(keys: list[dict]) -> str: + """Iterate through list of keys to check for correct one. + """ + for k in keys: + try: + return _decrypt(ciphertext=k['ciphertext'], key=k['encrypted_key']) + except InvalidTag: + continue + raise DotEnvVaultError('INVALID_DOTENV_KEY: Key must be valid.') + \ No newline at end of file diff --git a/src/dotenv_vault/test_vault.py b/src/dotenv_vault/test_vault.py new file mode 100644 index 0000000..9f97a94 --- /dev/null +++ b/src/dotenv_vault/test_vault.py @@ -0,0 +1,60 @@ +from io import StringIO +import os +import unittest + +from dotenv.main import DotEnv + +import dotenv_vault.main as vault + +class TestParsing(unittest.TestCase): + TEST_KEYS = [ + # OK. + ["dotenv://:key_0dec82bea24ada79a983dcc11b431e28838eae59a07a8f983247c7ca9027a925@dotenv.local/vault/.env.vault?environment=development", + True, "DOTENV_VAULT_DEVELOPMENT"], + + # Key too short (must be 64 characters + prefix). + ["dotenv://:key_1234@dotenv.org/vault/.env.vault?environment=production", + False, "DOTENV_VAULT_PRODUCTION"], + + # Missing key value. + ["dotenv://dotenv.org/vault/.env.vault?environment=production", + False, "DOTENV_VAULT_PRODUCTION"], + + # Missing environment. + ["dotenv://:key_1234@dotenv.org/vault/.env.vault", False, ""] + ] + + def test_key_parsing(self): + for test in self.TEST_KEYS: + dotenv_key, should_pass, environment_key_check = test + old_dotenv_key = os.environ.get("DOTENV_KEY") + os.environ["DOTENV_KEY"] = dotenv_key + try: + key, environment_key = vault.parse_key(dotenv_key) + self.assertTrue(should_pass) + self.assertEqual(environment_key, environment_key_check) + except Exception as exc: + self.assertFalse(should_pass) + finally: + os.unsetenv("DOTENV_KEY") + if old_dotenv_key: + os.environ["DOTENV_KEY"] = old_dotenv_key + + PARSE_TEST_KEY = "dotenv://:key_0dec82bea24ada79a983dcc11b431e28838eae59a07a8f983247c7ca9027a925@dotenv.local/vault/.env.vault?environment=development" + + PARSE_TEST_VAULT = """# .env.vault (generated with npx dotenv-vault local build) +DOTENV_VAULT_DEVELOPMENT="H2A2wOUZU+bjKH3kTpeua9iIhtK/q7/VpAn+LLVNnms+CtQ/cwXqiw==" +""" + + def test_vault_parsing(self): + old_dotenv_key = os.environ.get("DOTENV_KEY") + os.environ["DOTENV_KEY"] = self.PARSE_TEST_KEY + try: + stream = vault.parse_vault(StringIO(self.PARSE_TEST_VAULT)) + dotenv = DotEnv(dotenv_path=".env.vault", stream=stream) + self.assertEqual(dotenv.dict().get("HELLO"), "world") + finally: + os.unsetenv("DOTENV_KEY") + if old_dotenv_key: + os.environ["DOTENV_KEY"] = old_dotenv_key + \ No newline at end of file diff --git a/src/dotenv_vault/vault.py b/src/dotenv_vault/vault.py deleted file mode 100644 index 34ab26a..0000000 --- a/src/dotenv_vault/vault.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -import os -import io - -from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.exceptions import InvalidTag -from base64 import b64decode -from urllib.parse import urlparse, parse_qsl -from dotenv.main import DotEnv, find_dotenv - - -class DotEnvVaultError(Exception): - pass - - -class DotEnvVault(): #vault stuff - def __init__(self) -> None: - self.dotenv_key = os.environ.get('DOTENV_KEY') - - - def parsed_vault(self, dotenv_path: str) -> bytes: - """ - Parse information from DOTENV_KEY, and decrypt vault key. - """ - if self.dotenv_key is None: raise DotEnvVaultError("NOT_FOUND_DOTENV_KEY: Cannot find ENV['DOTENV_KEY']") - - # if dotenv_path is not present, then it will try to find default .env.vault file - env_vault_path = dotenv_path if dotenv_path else find_dotenv(filename='.env.vault', usecwd=True) - - if not env_vault_path: - raise DotEnvVaultError("ENV_VAULT_NOT_FOUND: .env.vault is not present.") - - keys = [] - dotenv_keys = [i.strip() for i in self.dotenv_key.split(',')] - for _key in dotenv_keys: - # parse DOTENV_KEY, format is a URI - uri = urlparse(_key) - # Get encrypted key - key = uri.password - # Get environment from query params. - params = dict(parse_qsl(uri.query)) - vault_environment = params.get('environment') - - if not vault_environment: - raise DotEnvVaultError('INVALID_DOTENV_KEY: Missing environment part') - - # Getting ciphertext from correct environment in .env.vault - environment_key = f'DOTENV_VAULT_{vault_environment.upper()}' - - # use python-dotenv library class. - dotenv = DotEnv(dotenv_path=env_vault_path) - ciphertext = dotenv.dict().get(environment_key) - - if not ciphertext: - raise DotEnvVaultError(f"NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment {environment_key} in your .env.vault file. Run 'npx dotenv-vault build' to include it.") - - keys.append({ - 'encrypted_key': key, - 'ciphertext': ciphertext - }) - - decrypted = self._key_rotation(keys=keys) - return self._to_text_stream(decrypted) - - - def _decrypt(self, ciphertext: str, key: str) -> bytes: - """ - decrypt method will decrypt via AES-GCM - return: decrypted keys in bytes - """ - _key = key[4:] - if len(_key) < 64: - raise DotEnvVault('INVALID_DOTENV_KEY: Key part must be 64 characters long (or more)') - - _key = bytes.fromhex(_key) - ciphertext = b64decode(ciphertext) - - aesgcm = AESGCM(_key) - return aesgcm.decrypt(ciphertext[:12], ciphertext[12:], b'') - - def _to_text_stream(self, decrypted_obj: bytes) -> io.StringIO: - """ - convert decrypted object (in bytes) to io.StringIO format. - Python-dotenv is expecting stream to be text stream (such as `io.StringIO`). - return: io.StringIO - """ - decoded_str = decrypted_obj.decode('utf-8') - return io.StringIO(decoded_str) - - def _key_rotation(self, keys: list[dict]) -> str: - """ - Iterate through list of keys to check for correct one. - """ - _len = len(keys) - for i, k in enumerate(keys): - try: - return self._decrypt(ciphertext=k['ciphertext'], key=k['encrypted_key']) - except InvalidTag: - if i + 1 >= _len: # exhaust all keys - raise DotEnvVaultError('INVALID_DOTENV_KEY: Key must be valid.') - else: - continue