Merge pull request #36 from pilosus/cli-tool

Piny CLI tool added (#35)

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+v0.6.0 (unreleased)
+...................
+* Add CLI utility (#35) by @pilosus
+
 v0.5.2 (2019-06-17)
 ...................
 * Fix ``Help`` section in ``README.rst`` (#31) by @pilosus

README.rst

@@ -10,7 +10,8 @@ Piny
 Keep your app's configuration in a YAML file.
 Mark up sensitive data in the config as *environment variables*.
 Set environment variables on application deployment.
-Now let the *Piny* load your config and interpolate environment variables in it.
+Now let the *Piny* load your config and substitute environment variables
+in it with their values.
 
 Piny is developed with Docker and Kubernetes in mind,
 though it's not limited to any deployment system.
@@ -79,6 +80,15 @@ Both strict and default matchers produce ``None`` value if environment variable
 matched is not set in the system (and no default syntax used in the case of
 default matcher).
 
+Piny also comes with *command line utility* that works both with files and standard
+input and output:
+
+.. code-block:: bash
+
+  $ export PASSWORD=mySecretPassword
+  $ echo "db: \${PASSWORD}" | piny
+  db: mySecretPassword
+
 
 Validation
 ----------

docs/conf.py

@@ -30,7 +30,8 @@
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
-    'sphinx.ext.viewcode'
+    'sphinx.ext.viewcode',
+    'sphinx_click.ext',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/index.rst

@@ -13,7 +13,7 @@ Piny: envs interpolation for config files
 - Keep your app's configuration in a YAML file.
 - Mark up sensitive data in config as *environment variables*.
 - Set environment variables on application deployment.
-- Let *Piny* load your configuration file and interpolate environment variables in it.
+- Let *Piny* load your configuration file and substitute environment variables with their values.
 
 Piny is developed with Docker and Kubernetes in mind,
 though it's not limited to any deployment system.
@@ -31,6 +31,19 @@ Then load your config with *Piny*:
 
 .. literalinclude:: code/simple_yaml_loader.py
 
+
+CLI utility
+-----------
+
+Piny's also got a command line tool working both with files and standard input and output:
+
+.. code-block:: bash
+
+  $ export PASSWORD=mySecretPassword
+  $ echo "db: \${PASSWORD}" | piny
+  db: mySecretPassword
+
+
 Rationale
 ---------
 

docs/integration.rst

@@ -1,7 +1,6 @@
 Integration Examples
 ====================
 
-
 Flask
 -----
 
@@ -12,3 +11,9 @@ aiohttp
 -------
 
 .. literalinclude:: code/aiohttp_integration.py
+
+
+Command line
+------------
+
+TODO

docs/requirements.txt

@@ -1,2 +1,3 @@
 Sphinx==2.1.1
 sphinx-rtd-theme==0.4.3
+sphinx-click==2.2.0

docs/usage.rst

@@ -11,7 +11,7 @@ is parsed and validated.
 Loaders
 -------
 
-As for now, *Piny* supports the only loader class called ``YamlLoader``.
+``YamlLoader`` loader class is dedicated for use in Python applications.
 Based on `PyYAML`_, it parses YAML files, (arguably) the most beautiful
 file format for configuration files!
 
@@ -28,6 +28,10 @@ Basic loader usage is the following.
 
 .. literalinclude:: code/simple_yaml_loader.py
 
+``YamlStreamLoader`` class primary use is Piny CLI tool (see :ref:`usage-cli-docs`).
+But it also can be used interchargably with ``YamlLoader`` whenever IO streams
+are used instead of file paths.
+
 .. automodule:: piny.loaders
     :members:
     :undoc-members:
@@ -120,3 +124,20 @@ Both exceptions inherit from the ``ConfigError``.
     :members:
     :undoc-members:
     :show-inheritance:
+
+
+.. _usage-cli-docs:
+
+Command line utility
+--------------------
+
+Piny comes with CLI tool that substitutes the values of environment variables
+in input file or ``stdin`` and write result to an output file or ``stdout``.
+Piny CLI utility is somewhat similar to ``GNU/gettext`` `envsubst`_ but works
+with files too.
+
+.. click:: piny.cli:cli
+   :prog: piny
+   :show-nested:
+
+.. _envsubst: https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html

piny/__init__.py

@@ -1,4 +1,4 @@
 from .errors import ConfigError, LoadingError, ValidationError
-from .loaders import YamlLoader
+from .loaders import YamlLoader, YamlStreamLoader
 from .matchers import Matcher, MatcherWithDefaults, StrictMatcher
 from .validators import MarshmallowValidator, PydanticValidator, TrafaretValidator

piny/cli.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+import click
+import yaml
+
+from .loaders import YamlStreamLoader
+from .matchers import MatcherWithDefaults, StrictMatcher
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+
+@click.command(context_settings=CONTEXT_SETTINGS)
+@click.argument("input", required=False, type=click.File("r"))
+@click.argument("output", required=False, type=click.File("w"))
+@click.option(
+    "--strict/--no-strict", default=True, help="Enable or disable strict matcher"
+)
+def cli(input, output, strict) -> Any:
+    """
+    Substitute environment variables with their values.
+
+    Read INPUT, find environment variables in it,
+    substitute them with their values and write to OUTPUT.
+
+    INPUT and OUTPUT can be files or standard input and output respectively.
+    With no INPUT, or when INPUT is -, read standard input.
+    With no OUTPUT, or when OUTPUT is -, write to standard output.
+
+    Examples:
+
+    \b
+    piny input.yaml output.yaml
+    piny - output.yaml
+    piny input.yaml -
+    tail -n 12 input.yaml | piny > output.yaml
+    """
+    stdin = click.get_text_stream("stdin")
+    stdout = click.get_text_stream("stdout")
+
+    config = YamlStreamLoader(
+        stream=input or stdin, matcher=StrictMatcher if strict else MatcherWithDefaults
+    ).load()
+
+    yaml.dump(config, output or stdout)

piny/loaders.py

@@ -1,4 +1,4 @@
-from typing import Any, Type
+from typing import IO, Any, Type, Union
 
 import yaml
 
@@ -18,8 +18,8 @@ class YamlLoader:
 
     def __init__(
         self,
-        *,
         path: str,
+        *,
         matcher: Type[Matcher] = MatcherWithDefaults,
         validator: Type[Validator] = None,
         schema: Any = None,
@@ -62,3 +62,37 @@ def load(self, **params) -> Any:
                 data=load, **params
             )
         return load
+
+
+class YamlStreamLoader(YamlLoader):
+    """
+    YAML configuration loader for IO streams, e.g. file objects or stdin
+    """
+
+    def __init__(
+        self,
+        stream: Union[str, IO[str]],
+        *,
+        matcher: Type[Matcher] = MatcherWithDefaults,
+        validator: Type[Validator] = None,
+        schema: Any = None,
+        **schema_params,
+    ) -> None:
+        self.stream = stream
+        self.matcher = matcher
+        self.validator = validator
+        self.schema = schema
+        self.schema_params = schema_params
+
+    def load(self, **params) -> Any:
+        self._init_resolvers()
+        try:
+            load = yaml.load(self.stream, Loader=self.matcher)
+        except yaml.YAMLError as e:
+            raise LoadingError(origin=e, reason=str(e))
+
+        if (self.validator is not None) and (self.schema is not None):
+            return self.validator(self.schema, **self.schema_params).load(
+                data=load, **params
+            )
+        return load

requirements.txt

@@ -1,5 +1,6 @@
 # Package dependencies
 PyYAML==5.1.1
+Click==7.0
 marshmallow==2.19.4
 pydantic==0.28
 trafaret==1.2.0

setup.py

@@ -46,10 +46,14 @@
     packages=find_packages(exclude=['tests']),
     install_requires=[
         'PyYAML>=5.1',
+        'Click>=7.0'
     ],
     extras_require={
         'pydantic': ['pydantic>=0.28'],
         'marshmallow': ['marshmallow>=2.19.3'],
         'trafaret': ['trafaret>=1.2.0'],
-    }
+    },
+    entry_points={
+          'console_scripts': ['piny = piny.cli:cli']
+    },
 )

tests/test_cli.py

@@ -0,0 +1,51 @@
+from unittest import mock
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from piny import LoadingError
+from piny.cli import cli
+
+from . import config_directory
+
+
+def test_cli_input_stdin_output_stdout():
+    runner = CliRunner()
+    with mock.patch("piny.matchers.StrictMatcher.constructor") as expand_mock:
+        expand_mock.return_value = "MySecretPassword"
+        result = runner.invoke(cli, input="password: ${DB_PASSWORD}")
+
+        assert result.exit_code == 0
+        assert result.stdout == "password: MySecretPassword\n"
+
+
+def test_cli_input_file_output_file():
+    runner = CliRunner()
+    with open(config_directory.joinpath("db.yaml"), "r") as f:
+        input_lines = f.readlines()
+
+    with runner.isolated_filesystem():
+        with open("input.yaml", "w") as input_fd:
+            input_fd.writelines(input_lines)
+
+        with mock.patch("piny.matchers.StrictMatcher.constructor") as expand_mock:
+            expand_mock.return_value = "MySecretPassword"
+            result = runner.invoke(cli, ["input.yaml", "output.yaml"])
+
+            with open("output.yaml", "r") as output_fd:
+                output_lines = output_fd.readlines()
+
+            assert result.exit_code == 0
+            assert "password: MySecretPassword" in map(
+                lambda x: x.strip(), output_lines
+            )
+
+
+def test_cli_fail():
+    runner = CliRunner()
+    with mock.patch("piny.loaders.yaml.load") as loader_mock:
+        loader_mock.side_effect = yaml.YAMLError("Oops!")
+        result = runner.invoke(cli, input="password: ${DB_PASSWORD}")
+        assert result.exit_code == 1
+        assert type(result.exception) == LoadingError

tests/test_validators.py

@@ -12,6 +12,7 @@
     TrafaretValidator,
     ValidationError,
     YamlLoader,
+    YamlStreamLoader,
 )
 
 from . import config_directory, config_map
@@ -109,6 +110,24 @@ def test_marshmallow_validator_fail(name):
         ).load(many=False)
 
 
+@pytest.mark.parametrize("name", ["db"])
+def test_marshmallow_validator_stream_success(name):
+    with mock.patch("piny.matchers.StrictMatcher.constructor") as expand_mock:
+        expand_mock.return_value = config_map[name]
+
+        with open(config_directory.joinpath("{}.yaml".format(name)), "r") as fd:
+            config = YamlStreamLoader(
+                stream=fd,
+                matcher=StrictMatcher,
+                validator=MarshmallowValidator,
+                schema=MarshmallowConfig,
+            ).load()
+
+            assert config[name]["host"] == "db.example.com"
+            assert config[name]["login"] == "user"
+            assert config[name]["password"] == config_map[name]
+
+
 @pytest.mark.parametrize("name", ["db"])
 def test_trafaret_validator_success(name):
     with mock.patch("piny.matchers.StrictMatcher.constructor") as expand_mock: