From 938f82005e10c6d4365793c81b9592cbdfe3ba67 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 May 2019 13:18:17 +0200 Subject: [PATCH] feat(config): added global config manager (#533) --- conftest.py | 4 +- renku/api/__init__.py | 2 +- renku/api/client.py | 7 +- renku/api/config.py | 113 +++++++++++++++++++++++++++++ renku/cli/__init__.py | 2 +- renku/cli/_config.py | 160 ------------------------------------------ renku/cli/_version.py | 2 +- renku/cli/config.py | 30 +++++--- renku/cli/endpoint.py | 53 -------------- renku/cli/env.py | 41 ----------- setup.py | 2 +- tests/test_cli.py | 52 ++++++++++++-- 12 files changed, 191 insertions(+), 277 deletions(-) create mode 100644 renku/api/config.py delete mode 100644 renku/cli/_config.py delete mode 100644 renku/cli/endpoint.py delete mode 100644 renku/cli/env.py diff --git a/conftest.py b/conftest.py index fe5b0a2766..47c38e9354 100644 --- a/conftest.py +++ b/conftest.py @@ -45,7 +45,7 @@ def instance_path(renku_path, monkeypatch): @pytest.fixture() def runner(monkeypatch): """Create a runner on isolated filesystem.""" - from renku.cli._config import RENKU_HOME + from renku.api.config import RENKU_HOME monkeypatch.setenv('RENKU_CONFIG', RENKU_HOME) return CliRunner() @@ -75,7 +75,7 @@ def generate(args=('update', ), cwd=None, **streams): @pytest.fixture() def isolated_runner(monkeypatch): """Create a runner on isolated filesystem.""" - from renku.cli._config import RENKU_HOME + from renku.api.config import RENKU_HOME monkeypatch.setenv('RENKU_CONFIG', RENKU_HOME) runner_ = CliRunner() with runner_.isolated_filesystem(): diff --git a/renku/api/__init__.py b/renku/api/__init__.py index 4373193511..981b3a0a28 100644 --- a/renku/api/__init__.py +++ b/renku/api/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2017-2018 - Swiss Data Science Center (SDSC) +# Copyright 2017-2019 - Swiss Data Science Center (SDSC) # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and # Eidgenössische Technische Hochschule Zürich (ETHZ). # diff --git a/renku/api/client.py b/renku/api/client.py index 82e75057d7..b2df23be32 100644 --- a/renku/api/client.py +++ b/renku/api/client.py @@ -19,6 +19,7 @@ import attr +from .config import ConfigManagerMixin from .datasets import DatasetsApiMixin from .repository import PathMixin, RepositoryApiMixin from .storage import StorageApiMixin @@ -26,10 +27,8 @@ @attr.s class LocalClient( - PathMixin, - StorageApiMixin, - RepositoryApiMixin, - DatasetsApiMixin, + PathMixin, StorageApiMixin, RepositoryApiMixin, DatasetsApiMixin, + ConfigManagerMixin ): """A low-level client for communicating with a local Renku repository. diff --git a/renku/api/config.py b/renku/api/config.py new file mode 100644 index 0000000000..07de27db48 --- /dev/null +++ b/renku/api/config.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Client for handling a configuration.""" +import configparser +import fcntl +import os +from pathlib import Path + +import attr +import click + +APP_NAME = 'Renku' +"""Application name for storing configuration.""" + +RENKU_HOME = '.renku' +"""Project directory name.""" + + +def print_app_config_path(ctx, param, value): + """Print application config path.""" + if not value or ctx.resilient_parsing: + return + click.echo(ConfigManagerMixin().config_path) + ctx.exit() + + +def default_config_dir(): + """Return default config directory.""" + return click.get_app_dir(APP_NAME, force_posix=True) + + +@attr.s +class ConfigManagerMixin: + """Client for handling global configuration.""" + + config_dir = attr.ib(default=default_config_dir(), converter=str) + config_name = attr.ib(default='renku.ini', converter=str) + + _lock = attr.ib(default=None) + + def __enter__(self): + """Acquire a lock file.""" + lock_name = '{0}/{1}.lock'.format(self.config_dir, self.config_name) + locked_file_descriptor = open(lock_name, 'w+') + fcntl.lockf(locked_file_descriptor, fcntl.LOCK_EX) + self._lock = locked_file_descriptor + + def __exit__(self, type, value, traceback): + """Release lock file.""" + self._lock.close() + + @property + def config_path(self): + """Renku config path.""" + config = Path(self.config_dir) + if not config.exists(): + config.mkdir() + + return config / Path(self.config_name) + + def load_config(self): + """Loads global configuration object.""" + config = configparser.ConfigParser() + config.read(str(self.config_path)) + return config + + def store_config(self, config): + """Persists global configuration object.""" + os.umask(0) + fd = os.open(str(self.config_path), os.O_CREAT | os.O_WRONLY, 0o600) + with open(fd, 'w') as file: + config.write(file) + + def get_value(self, section, key): + """Get value from specified section and key.""" + config = self.load_config() + return config.get(section, key, fallback=None) + + def set_value(self, section, key, value): + """Set value to specified section and key.""" + config = self.load_config() + if section in config: + config[section][key] = value + else: + config[section] = {key: value} + + self.store_config(config) + return config + + +def get_config(client, write_op, is_global): + """Get configuration object.""" + if is_global: + return client + + if write_op: + return client.repo.config_writer() + return client.repo.config_reader() diff --git a/renku/cli/__init__.py b/renku/cli/__init__.py index 30643dedd9..0c4b9cebb7 100644 --- a/renku/cli/__init__.py +++ b/renku/cli/__init__.py @@ -82,8 +82,8 @@ import yaml from ..api.client import LocalClient +from ..api.config import RENKU_HOME, default_config_dir, print_app_config_path from ..api.repository import default_path -from ._config import RENKU_HOME, default_config_dir, print_app_config_path from ._exc import IssueFromTraceback from ._options import install_completion, option_use_external_storage from ._version import check_version, print_version diff --git a/renku/cli/_config.py b/renku/cli/_config.py deleted file mode 100644 index 556a81585b..0000000000 --- a/renku/cli/_config.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017-2019 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Configuration utilities.""" - -import errno -import os -from functools import update_wrapper - -import click -import yaml - -from renku._compat import Path - -from ._options import Endpoint - -APP_NAME = 'Renku' -"""Application name for storing configuration.""" - -RENKU_HOME = '.renku' -"""Project directory name.""" - -# Register Endpoint serializer -yaml.add_representer( - Endpoint, lambda dumper, data: dumper.represent_str(str(data)) -) - - -def default_config_dir(): - """Return default config directory.""" - return click.get_app_dir(APP_NAME) - - -def config_path(path=None, final=False): - """Return config path.""" - if final and path: - return path - - if path is None: - path = default_config_dir() - try: - os.makedirs(path) - except OSError as e: # pragma: no cover - if e.errno != errno.EEXIST: - raise - return os.path.join(path, 'config.yml') - - -def read_config(path=None, final=False): - """Read Renku configuration.""" - try: - with open(config_path(path, final=final), 'r') as configfile: - return yaml.safe_load(configfile) or {} - except FileNotFoundError: - return {} - - -def write_config(config, path, final=False): - """Write Renku configuration.""" - with open(config_path(path, final=final), 'w+') as configfile: - yaml.dump(config, configfile, default_flow_style=False) - - -def config_load(ctx, param, value): - """Print application config path.""" - if ctx.obj is None: - ctx.obj = {} - - ctx.obj['config_path'] = value - ctx.obj['config'] = read_config(value) - return value - - -def with_config(f): - """Add config to function.""" - # keep it. - - @click.pass_context - def new_func(ctx, *args, **kwargs): - # Invoked with custom config: - if 'config' in kwargs: - return ctx.invoke(f, *args, **kwargs) - - if ctx.obj is None: - ctx.obj = {} - - config = ctx.obj['config'] - - project_enabled = not ctx.obj.get('no_project', False) - project_config_path = get_project_config_path() - - if project_enabled and project_config_path: - project_config = read_config(project_config_path) - config['project'] = project_config - result = ctx.invoke(f, config, *args, **kwargs) - project_config = config.pop('project', None) - if project_config: - if not project_config_path: - raise RuntimeError('Invalid config update') - write_config(project_config, path=project_config_path) - write_config(config, path=ctx.obj['config_path']) - if project_config is not None: - config['project'] = project_config - return result - - return update_wrapper(new_func, f) - - -def print_app_config_path(ctx, param, value): - """Print application config path.""" - if not value or ctx.resilient_parsing: - return - click.echo(config_path(os.environ.get('RENKU_CONFIG'))) - ctx.exit() - - -def create_project_config_path( - path, mode=0o777, parents=False, exist_ok=False -): - """Create new project configuration folder.""" - # FIXME check default directory mode - project_path = Path(path).absolute().joinpath(RENKU_HOME) - project_path.mkdir(mode=mode, parents=parents, exist_ok=exist_ok) - return str(project_path) - - -def get_project_config_path(path=None): - """Return project configuration folder if exist.""" - project_path = Path(path or '.').absolute().joinpath(RENKU_HOME) - if project_path.exists() and project_path.is_dir(): - return str(project_path) - - -def find_project_config_path(path=None): - """Find project config path.""" - path = Path(path) if path else Path.cwd() - abspath = path.absolute() - - project_path = get_project_config_path(abspath) - if project_path: - return project_path - - for parent in abspath.parents: - project_path = get_project_config_path(parent) - if project_path: - return project_path diff --git a/renku/cli/_version.py b/renku/cli/_version.py index 2552753da9..91b2a0f176 100644 --- a/renku/cli/_version.py +++ b/renku/cli/_version.py @@ -127,7 +127,7 @@ def dump(self, app_name): def _check_version(): """Check renku version.""" - from ._config import APP_NAME + from ..api.config import APP_NAME if VersionCache.load(APP_NAME).is_fresh: return diff --git a/renku/cli/config.py b/renku/cli/config.py index 13fa959e73..9b9d3bad28 100644 --- a/renku/cli/config.py +++ b/renku/cli/config.py @@ -38,8 +38,12 @@ https://registry.gitlab.com/demo/demo """ +import configparser import click +from click import BadParameter + +from renku.api.config import get_config from ._client import pass_local_client @@ -55,14 +59,24 @@ def _split_section_and_key(key): @click.command() @click.argument('key', required=True) @click.argument('value', required=False, default=None) +@click.option( + '--global', + 'is_global', + is_flag=True, + help='Store to global configuration.' +) @pass_local_client -def config(client, key, value): - """Get and set Renku repository and global options.""" - if value is None: - cfg = client.repo.config_reader() - click.echo(cfg.get_value(*_split_section_and_key(key))) - else: - with client.repo.config_writer() as cfg: +def config(client, key, value, is_global): + """Manage configuration options.""" + write_op = value is not None + config_ = get_config(client, write_op, is_global) + if write_op: + with config_: section, config_key = _split_section_and_key(key) - cfg.set_value(section, config_key, value) + config_.set_value(section, config_key, value) click.echo(value) + else: + try: + click.echo(config_.get_value(*_split_section_and_key(key))) + except configparser.NoSectionError: + raise BadParameter('Requested configuration not found') diff --git a/renku/cli/endpoint.py b/renku/cli/endpoint.py deleted file mode 100644 index 148c6e7425..0000000000 --- a/renku/cli/endpoint.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017-2019 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Manage set of platform API endpoints.""" - -import click - -from ._config import with_config -from ._options import argument_endpoint - - -@click.group(invoke_without_command=True) -@click.option('-v', '--verbose', count=True) -@with_config -@click.pass_context -def endpoint(ctx, config, verbose): - """Manage set of platform API endpoints.""" - if ctx.invoked_subcommand is None: - # TODO default_endpoint = config.get('core', {}).get('default') - for endpoint, values in config.get('endpoints', {}).items(): - # TODO is_default = default_endpoint == endpoint - if not verbose: - click.echo(endpoint) - else: - click.echo( - '{endpoint}\t{url}'.format( - endpoint=endpoint, url=values.get('url', '') - ) - ) - - -@endpoint.command(name='set-default') -@argument_endpoint -@with_config -@click.pass_context -def set_default(ctx, config, endpoint): - """Set endpoint as default.""" - config.setdefault('core', {}) - config['core']['default'] = endpoint diff --git a/renku/cli/env.py b/renku/cli/env.py deleted file mode 100644 index 5f1c1f20bc..0000000000 --- a/renku/cli/env.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017-2019 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Print useful ``RENKU`` environment variables that can be sourced.""" - -import click - -from ._config import with_config -from ._options import default_endpoint - - -@click.command() -@click.argument('endpoint', required=False, callback=default_endpoint) -@with_config -def env(config, endpoint): - """Print RENKU environment variables. - - Run this command to configure your Renku client: - - $ eval "$(renku env)" - - """ - access_token = config['endpoints'][endpoint]['token']['access_token'] - click.echo('export {0}={1}'.format('RENKU_ENDPOINT', endpoint)) - click.echo('export {0}={1}'.format('RENKU_ACCESS_TOKEN', access_token)) - click.echo('# Run this command to configure your Renku client:') - click.echo('# eval "$(renku env)"') diff --git a/setup.py b/setup.py index 1e298fdd70..367cd7f5f9 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ tests_require = [ 'check-manifest>=0.37', - 'coverage>=4.0', + 'coverage>=4.5.3', 'flake8>=3.5', 'freezegun>=0.3.9', 'isort==4.3.4', diff --git a/tests/test_cli.py b/tests/test_cli.py index dd75cbd443..5f73f005ec 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -51,12 +51,9 @@ def test_help(arg, runner): def test_config_path(runner): """Test config path.""" - from renku.cli._config import RENKU_HOME - result = runner.invoke(cli.cli, ['--config-path']) output = result.output.split('\n')[0] - assert 'config.yml' in output - assert RENKU_HOME in output + assert 'renku.ini' in output def test_show_context(runner): @@ -923,17 +920,18 @@ def test_image_pull(runner, project): result = runner.invoke( cli.cli, ['config', 'registry', 'http://demo:demo@global.example.com'] ) + assert 'http://demo:demo@global.example.com\n' == result.output assert 0 == result.exit_code cmd = ['image', 'pull', '--no-auto-login'] result = runner.invoke(cli.cli, cmd) assert 'global.example.com' in result.output assert 1 == result.exit_code - result = runner.invoke( cli.cli, ['config', 'origin.registry', 'http://demo:demo@origin.example.com'] ) + assert 'http://demo:demo@origin.example.com\n' == result.output assert 0 == result.exit_code cmd = ['image', 'pull', '--no-auto-login'] @@ -1093,3 +1091,47 @@ def test_input_directory(runner, project, run): str(p.relative_to(cwd)) for p in inputs.rglob('*') if p.name != '.gitkeep' ) == set(result.output.strip().split('\n')) + + +def test_config_manager_creation(client): + """Check creation of configuration file.""" + path_ = client.config_path + assert str(path_).endswith('renku.ini') + config = client.load_config() + client.store_config(config) + assert path_.exists() + + +def test_config_manager_set_value(client): + """Check writing to configuration.""" + client.set_value('zenodo', 'secret', 'my-secret') + config = client.load_config() + assert config.get('zenodo', 'secret') == 'my-secret' + + +def test_config_load_get_value(client): + """Check reading from configuration.""" + client.set_value('zenodo', 'secret', 'my-secret') + secret = client.get_value('zenodo', 'secret') + assert secret == 'my-secret' + + secret = client.get_value('zenodo2', 'secret') + assert secret is None + + secret = client.get_value('zenodo', 'not-secret') + assert secret is None + + +def test_config_manager_cli(client, runner, project): + """Check config command for global cfg.""" + result = runner.invoke( + cli.cli, [ + 'config', 'registry', 'http://demo:demo@global.example.com', + '--global' + ] + ) + assert 'http://demo:demo@global.example.com\n' == result.output + assert 0 == result.exit_code + + value = client.get_value('renku', 'registry') + assert 'http://demo:demo@global.example.com' == value