Permalink
Browse files

Introduce a 'central' .cs.json for cs command. (#1074)

  • Loading branch information...
yueri authored and pschorf committed Feb 7, 2019
1 parent d09c449 commit 8d305f9a1035ea946b9446acfcb314666d76b239
Showing with 100 additions and 26 deletions.
  1. +6 −1 cli/README.md
  2. +1 −1 cli/cook/cli.py
  3. +27 −9 cli/cook/configuration.py
  4. +6 −5 cli/cook/subcommands/config.py
  5. +22 −0 integration/tests/cook/cli.py
  6. +38 −10 integration/tests/cook/test_cli.py
@@ -17,7 +17,12 @@ Custom defaults may be provided for all commands.
Multiple clusters are supported via configuration.

In order to use the Cook CLI, you’ll need a configuration file.
`cs` looks first for a `.cs.json` file in the current directory, and then for a `.cs.json` file in your home directory.
`cs` uses a default `.cs.json` file located in either `${DIR}/.cs.json`, `${DIR}/../.cs.json`, or `${DIR}/../config/.cs.json`, where `${DIR}` is the location of the `cs` executable.
This allows you to roll out configuration changes (such as adding a new cluster) without updating the configuration files of all end users.
If a default `.cs.json` file is not found, an empty configuration is used.

Users can also add and overwrite any properties in the default `.cs.json` file using a local `.cs.json` file.
`cs` first looks for this `.cs.json` file in your current directory, and then for this `.cs.json` file in your home directory.
The path to this file may also be provided manually via the command line with the `--config` option.

There is a sample `.cs.json` file included in this directory, which looks something like this:
@@ -85,7 +85,7 @@ def run(args):
if action is None:
parser.print_help()
else:
config_map = configuration.load_config_with_defaults(config_path)
_, config_map = configuration.load_config_with_defaults(config_path)
try:
metrics.initialize(config_map)
metrics.inc('command.%s.runs' % action)
@@ -1,12 +1,19 @@
import json
import logging
import os
import sys

from cook.util import deep_merge

# Default locations to check for configuration files if one isn't given on the command line
DEFAULT_CONFIG_PATHS = ['.cs.json',
os.path.expanduser('~/.cs.json')]
# Base locations to check for configuration files, relative to the executable.
# Always tries to load these in.
BASE_CONFIG_PATHS = ['.cs.json',
'../.cs.json',
'../config/.cs.json']

# Additional locations to check for configuration files if one isn't given on the command line
ADDITIONAL_CONFIG_PATHS = ['.cs.json',
os.path.expanduser('~/.cs.json')]

DEFAULT_CONFIG = {'defaults': {},
'http': {'retries': 2,
@@ -44,10 +51,18 @@ def __load_first_json_file(paths):
return next(((p, c) for p, c in contents if c), (None, None))


def load_config(config_path):
def __load_base_config():
"""Loads the base configuration map."""
base_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
paths = [os.path.join(base_dir, path) for path in BASE_CONFIG_PATHS]
_, config = __load_first_json_file(paths)
return config


def __load_local_config(config_path):
"""
Loads the configuration map, using the provided config_path if not None,
otherwise, searching the default config paths for a valid JSON config file
otherwise, searching the additional config paths for a valid JSON config file
"""
if config_path:
if os.path.isfile(config_path):
@@ -56,18 +71,21 @@ def load_config(config_path):
else:
raise Exception(f'The configuration path specified ({config_path}) is not valid.')
else:
config_path, config = __load_first_json_file(DEFAULT_CONFIG_PATHS)
config_path, config = __load_first_json_file(ADDITIONAL_CONFIG_PATHS)

return config_path, config


def load_config_with_defaults(config_path=None):
"""Loads the configuration map to use, merging in the defaults"""
_, config = load_config(config_path)
base_config = __load_base_config()
base_config = base_config or {}
base_config = deep_merge(DEFAULT_CONFIG, base_config)
config_path, config = __load_local_config(config_path)
config = config or {}
config = deep_merge(DEFAULT_CONFIG, config)
config = deep_merge(base_config, config)
logging.debug(f'using configuration: {config}')
return config
return config_path, config


def add_defaults(action, defaults):
@@ -1,7 +1,7 @@
from cook.util import print_info

from cook import configuration, terminal
from cook.configuration import load_config
from cook.configuration import load_config_with_defaults


def get_in(dct, keys):
@@ -98,14 +98,13 @@ def config(_, args, config_path):
raise Exception(f'You can only provide a single key.')

keys = key[0].split('.')
config_path, config_map = load_config(config_path)

if not config_path:
raise Exception(f'Unable to locate configuration file.')
config_path, config_map = load_config_with_defaults(config_path)

if get:
return get_config_value(config_map, keys)
else:
if not config_path:
raise Exception(f'Unable to locate configuration file.')
return set_config_value(config_map, keys, value, config_path)


@@ -117,3 +116,5 @@ def register(add_parser, _):
parser.add_argument('key', nargs=1)
parser.add_argument('value', nargs='?')
return config


@@ -209,6 +209,28 @@ def __exit__(self, _, __, ___):
os.remove(self.path)


class temp_base_config_file:
"""
A context manager used to generate and subsequently delete a temporary
base config file for the CLI. Takes as input the config dictionary to use.
"""

def __init__(self, config):
# Get the location of the cs executable so we can add a default `.cs.json` file
cp = subprocess.run(args=['which', command()], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
base_exec = cp.stdout.decode("utf-8").rstrip('\n')
base_dir = os.path.dirname(os.path.abspath(base_exec))
self.path = os.path.join(base_dir, '.cs.json')
self.config = config

def __enter__(self):
write_json(self.path, self.config)
return self.path

def __exit__(self, _, __, ___):
os.remove(self.path)


def jobs(cook_url=None, jobs_flags=None, flags=None):
"""Invokes the jobs subcommand"""
args = f'jobs {jobs_flags}' if jobs_flags else 'jobs'
@@ -1308,6 +1308,34 @@ def test_submit_with_command_prefix(self):
self.assertEqual('export FOO=0; exit ${FOO:-1}', jobs[0]['command'])
self.assertEqual('success', jobs[0]['state'])

def test_base_config_file(self):
base_config = {'defaults': {'submit': {'command-prefix': 'export FOO=0; '}}, 'other': 'bar'}

with cli.temp_base_config_file(base_config) as base_config:
# Get entries in base config file
cp = cli.config_get('defaults.submit.command-prefix')
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('export FOO=0; \n', cli.decode(cp.stdout))

cp = cli.config_get('other')
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEquals('bar\n', cli.decode(cp.stdout))

# Overwrite defaults with specified config file
config = {'defaults': {'submit': {'command-prefix': 'export FOO=1; '}}}
with cli.temp_config_file(config) as path:
flags = '--config %s' % path

# Verify default config is overwritten
cp = cli.config_get('defaults.submit.command-prefix', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('export FOO=1; \n', cli.decode(cp.stdout))

# Verify other config is still loaded
cp = cli.config_get('other', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('bar\n', cli.decode(cp.stdout))

def test_config_command_basics(self):
config = {'defaults': {'submit': {'command-prefix': 'export FOO=0; '}}}
with cli.temp_config_file(config) as path:
@@ -1346,16 +1374,6 @@ def test_config_command_advanced(self):
with cli.temp_config_file(config) as path:
flags = '--config %s' % path

# Set an entry that doesn't exist
cp = cli.config_get('defaults.submit.command-prefix', flags=flags)
self.assertEqual(1, cp.returncode, cp.stderr)
self.assertIn('Configuration entry defaults.submit.command-prefix not found.', cli.decode(cp.stderr))
cp = cli.config_set('defaults.submit.command-prefix', '"export FOO=1; "', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
cp = cli.config_get('defaults.submit.command-prefix', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('export FOO=1; \n', cli.decode(cp.stdout))

# Set at the top level
cp = cli.config_get('foo', flags=flags)
self.assertEqual(1, cp.returncode, cp.stderr)
@@ -1371,6 +1389,16 @@ def test_config_command_advanced(self):
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('baz\n', cli.decode(cp.stdout))

# Set nested entry
cp = cli.config_get('new.nested.entry', flags=flags)
self.assertEqual(1, cp.returncode, cp.stderr)
self.assertIn('Configuration entry new.nested.entry not found.', cli.decode(cp.stderr))
cp = cli.config_set('new.nested.entry', '"exists"', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
cp = cli.config_get('new.nested.entry', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual('exists\n', cli.decode(cp.stdout))

# Set non-string types
cp = cli.config_set('int', '12345', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)

0 comments on commit 8d305f9

Please sign in to comment.