diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e08af8..6690536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # CUMIN CHANGELOG +## v1.1.0 (2017-09-21) + +### New features: +* Backends: add OpenStack backend ([T175711](https://phabricator.wikimedia.org/T175711)) + +### Bug Fixes: +* CLI: fix --version option +* Installation: fix data_files installation directory ([T174008](https://phabricator.wikimedia.org/T174008)) +* Transports: better handling of empty list ([T174911](https://phabricator.wikimedia.org/T174911)): + * BaseWorker: accept an empty list in the command setter. It's its default value, there is no point in forbidding a + client to set it to the same value. + * ClusterShellWorker: return immediately if there are no target hosts. +* Clustershell: make call to tqdm.write() explicit where to send the output, not relying on its default. + + ## v1.0.0 (2017-08-23) ### CLI breaking changes: diff --git a/README.md b/README.md index 8769e1d..ab7da2f 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,35 @@ Given that the `pyparsing` library used to define the grammar uses a BNF-like st specified above see directly the code in `cumin/backends/puppetdb.py`. +##### OpenStack + +This backend uses the OpenStack APIs to perform the query. The specific query language has this features: + +- Each query can specify multiple parameters to filter the hosts selection in the form `key:value`. +- The special `project` key allow to filter by the OpenStack project name: `project:project_name`. If not specified + all the visible and enabled projects will be queried. +- Any other `key:value` pair will be passed as is to the + [OpenStack list-servers API](https://developer.openstack.org/api-ref/compute/#list-servers). + Multiple filters can be added separated by space. The value can be enclosed in single or double quotes: + `name:"host1.*\.domain" image:UUID` +- By default the filters `status:ACTIVE` and `vm_state:ACTIVE` are also added, but will be overridden if specified in + the query. +- To mix multiple selections the general grammar must be used with multiple subqueries: + `O{project:project1} or O{project:project2}` +- The special query `*` is a shortcut to select all hosts in all OpenStack projects. +- See the example configuration in `doc/examples/config.yaml` for all the OpenStack-related parameters that can be set. + +``` +Backus-Naur form (BNF) of the grammar: + ::= "*" | + ::= | + ::= : +``` + +Given that the `pyparsing` library used to define the grammar uses a BNF-like style, for the details of the tokens not +specified above see directly the code in `cumin/backends/openstack.py`. + + ##### Direct The `direct` backend allow to use Cumin without any external dependency for the hosts selection. @@ -183,7 +212,7 @@ Is it also possible to build a Debian package using the `debian` branch, for exa The default configuration file for `cumin` CLI is expected to be found at `/etc/cumin/config.yaml`; the path can be changed via a command-line switch, `--config`. A commented example configuration is available in -`doc/examples//config.yaml`. +`doc/examples/config.yaml`. Cumin will also automatically load any aliases defined in a `aliases.yaml` file, if present in the same directory of the main configuration file. An aliases example file is available in `doc/examples/aliases.yaml` @@ -191,9 +220,9 @@ the main configuration file. An aliases example file is available in `doc/exampl ## CLI ### Usage - - cumin [OPTIONS] HOSTS COMMAND [COMMAND ...] - +``` +cumin [OPTIONS] HOSTS COMMAND [COMMAND ...] +``` ### OPTIONS For the full list of available optional arguments see `cumin --help`. @@ -224,18 +253,23 @@ Multiple commands will be executed sequentially. ## Running tests The `tox` utility, a wrapper around virtualenv, is used to run the test. To list the available environements: - - tox -l - +``` +tox -l +``` To run one: +``` +tox -e flake8 +``` +You can pass extra arguments to the underlying command: +``` +# Run only tests in a specific file: +tox -e unit -- -k test_puppetdb.py - tox -e flake8 - -You can pass extra arguments to the underlying command, for example to only run the tests in a specific file: - - tox -e unit -- -k test_puppetdb.py - +# Run only one specific test: +tox -e unit -- -k test_invalid_grammars +``` Also integration tests are available, but not run by default by tox. They depends on a running Docker instance. To run them: - - tox -e integration +``` +tox -e integration +``` diff --git a/cumin/backends/__init__.py b/cumin/backends/__init__.py index b85c19f..a8d0d37 100644 --- a/cumin/backends/__init__.py +++ b/cumin/backends/__init__.py @@ -59,22 +59,6 @@ def _parse_token(self, token): token -- a single token returned by the grammar parsing """ - @abstractmethod - def _open_subgroup(self): - """Is called when a subgroup is opened in the parsing of the query. - - Given that each backend has it's own grammar and parsing logic, keeping this in the abstract class to force - each backend to support subgrouping in the grammar for usability and coherence between backends. - """ - - @abstractmethod - def _close_subgroup(self): - """Is called when a subgroup is closed in the parsing of the query. - - Given that each backend has it's own grammar and parsing logic, keeping this in the abstract class to force - each backend to support subgrouping in the grammar for usability and coherence between backends. - """ - def _build(self, query_string): """Parse the query string according to the grammar and build the query for later execution. @@ -121,14 +105,14 @@ def _execute(self): return hosts def _open_subgroup(self): - """Required by BaseQuery.""" + """Handle subgroup opening.""" element = self._get_stack_element() element['parent'] = self.stack_pointer self.stack_pointer['children'].append(element) self.stack_pointer = element def _close_subgroup(self): - """Required by BaseQuery.""" + """Handle subgroup closing.""" self.stack_pointer = self.stack_pointer['parent'] @abstractmethod diff --git a/cumin/backends/openstack.py b/cumin/backends/openstack.py new file mode 100644 index 0000000..c48bca5 --- /dev/null +++ b/cumin/backends/openstack.py @@ -0,0 +1,165 @@ +"""OpenStack backend.""" +import pyparsing as pp + +from ClusterShell.NodeSet import NodeSet +from keystoneauth1 import session as keystone_session +from keystoneauth1.identity import v3 as keystone_identity +from keystoneclient.v3 import client as keystone_client +from novaclient import client as nova_client + +from cumin.backends import BaseQuery, InvalidQueryError + + +def grammar(): + """Define the query grammar. + + Some query examples: + - All hosts in all OpenStack projects: `*` + - All hosts in a specific OpenStack project: `project:project_name` + - Filter hosts using any parameter allowed by the OpenStack list-servers API: `name:host1 image:UUID` + See https://developer.openstack.org/api-ref/compute/#list-servers for more details. + Multiple filters can be added separated by space. The value can be enclosed in single or double quotes. + If the `project` key is not specified the hosts will be selected from all projects. + - To mix multiple selections the general grammar must be used with multiple subqueries: + `O{project:project1} or O{project:project2}` + + Backus-Naur form (BNF) of the grammar: + ::= "*" | + ::= | + ::= : + + Given that the pyparsing library defines the grammar in a BNF-like style, for the details of the tokens not + specified above check directly the code. + """ + quoted_string = pp.quotedString.copy().addParseAction(pp.removeQuotes) # Both single and double quotes are allowed + + # Key-value tokens: key:value + # All printables characters except the parentheses that are part of this or the global grammar + key = pp.Word(pp.alphanums + '-_.')('key') + all_but_par = ''.join([c for c in pp.printables if c not in ('(', ')', '{', '}')]) + value = (quoted_string | pp.Word(all_but_par))('value') + item = pp.Combine(key + ':' + value) + + # Final grammar, see the docstring for its BNF based on the tokens defined above + # Groups are used to split the parsed results for an easy access + return pp.Group(pp.Literal('*')('all')) | pp.OneOrMore(pp.Group(item)) + + +def _get_keystone_session(config, project=None): + """Return a new keystone session based on configuration. + + Arguments: + config -- a dictionary with the session configuration: auth_url, username, password + project -- a project to scope the session to. [optional, default: None] + """ + auth = keystone_identity.Password( + auth_url='{auth_url}/v3'.format(auth_url=config.get('auth_url', 'http://localhost:5000')), + username=config.get('username', 'username'), + password=config.get('password', 'password'), + project_name=project, + user_domain_id='default', + project_domain_id='default') + return keystone_session.Session(auth=auth) + + +def _get_nova_client(config, project): + """Return a new nova client tailored to the given project. + + Arguments: + config -- a dictionary with the session configuration: auth_url, username, password, nova_api_version, timeout + project -- a project to scope the session to. [optional, default: None] + """ + return nova_client.Client( + config.get('nova_api_version', '2'), + session=_get_keystone_session(config, project), + endpoint_type='public', + timeout=config.get('timeout', 10)) + + +class OpenStackQuery(BaseQuery): + """OpenStackQuery query builder. + + Query VMs deployed in an OpenStack infrastructure using the API. + """ + + grammar = grammar() + + def __init__(self, config, logger=None): + """Query constructor for the OpenStack backend. + + Arguments: according to BaseQuery interface + """ + super(OpenStackQuery, self).__init__(config, logger=logger) + self.openstack_config = self.config.get('openstack', {}) + self.search_project = None + self.search_params = OpenStackQuery._get_default_search_params() + + @staticmethod + def _get_default_search_params(): + """Return the default search parameters dictionary.""" + return {'status': 'ACTIVE', 'vm_state': 'ACTIVE'} + + def _build(self, query_string): + """Override parent class _build method to reset search parameters.""" + self.search_params = OpenStackQuery._get_default_search_params() + super(OpenStackQuery, self)._build(query_string) + + def _execute(self): + """Required by BaseQuery.""" + if self.search_project is None: + hosts = NodeSet() + for project in self._get_projects(): + hosts |= self._get_project_hosts(project) + else: + hosts = self._get_project_hosts(self.search_project) + + return hosts + + def _parse_token(self, token): + """Required by BaseQuery.""" + if not isinstance(token, pp.ParseResults): # pragma: no cover - this should never happen + raise InvalidQueryError('Expecting ParseResults object, got {type}: {token}'.format( + type=type(token), token=token)) + + token_dict = token.asDict() + self.logger.trace('Token is: {token_dict} | {token}'.format(token_dict=token_dict, token=token)) + + if 'key' in token_dict and 'value' in token_dict: + if token_dict['key'] == 'project': + self.search_project = token_dict['value'] + else: + self.search_params[token_dict['key']] = token_dict['value'] + elif 'all' in token_dict: + pass # nothing to do, search_project and search_params have the right defaults + else: # pragma: no cover - this should never happen + raise InvalidQueryError('Got unexpected token: {token}'.format(token=token)) + + def _get_projects(self): + """Yield the project names for all projects (except admin) from keystone API.""" + client = keystone_client.Client( + session=_get_keystone_session(self.openstack_config), timeout=self.openstack_config.get('timeout', 10)) + return (project.name for project in client.projects.list(enabled=True) if project.name != 'admin') + + def _get_project_hosts(self, project): + """Return a NodeSet with the list of matching hosts based for the project based on the search parameters. + + Arguments: + project -- the project name where to get the list of hosts + """ + client = _get_nova_client(self.openstack_config, project) + + domain = '' + domain_suffix = self.openstack_config.get('domain_suffix', None) + if domain_suffix is not None: + if domain_suffix[0] != '.': + domain = '.{suffix}'.format(suffix=domain_suffix) + else: + domain = domain_suffix + + return NodeSet.fromlist('{host}.{project}{domain}'.format(host=server.name, project=project, domain=domain) + for server in client.servers.list(search_opts=self.search_params)) + + +# Required by the backend auto-loader in cumin.grammar.get_registered_backends() +GRAMMAR_PREFIX = 'O' +query_class = OpenStackQuery # pylint: disable=invalid-name diff --git a/cumin/backends/puppetdb.py b/cumin/backends/puppetdb.py index 2352279..877a1e7 100644 --- a/cumin/backends/puppetdb.py +++ b/cumin/backends/puppetdb.py @@ -59,7 +59,7 @@ def grammar(): neg = pp.CaselessKeyword('not')('neg') operator = pp.oneOf(OPERATORS, caseless=True)('operator') # Comparison operators - quoted_string = pp.quotedString.addParseAction(pp.removeQuotes) # Both single and double quotes are allowed + quoted_string = pp.quotedString.copy().addParseAction(pp.removeQuotes) # Both single and double quotes are allowed # Parentheses lpar = pp.Literal('(')('open_subgroup') @@ -138,14 +138,14 @@ def category(self, value): self._category = value def _open_subgroup(self): - """Required by BaseQuery.""" + """Handle subgroup opening.""" token = PuppetDBQuery._get_grouped_tokens() token['parent'] = self.current_group self.current_group['tokens'].append(token) self.current_group = token def _close_subgroup(self): - """Required by BaseQuery.""" + """Handle subgroup closing.""" self.current_group = self.current_group['parent'] @staticmethod diff --git a/cumin/cli.py b/cumin/cli.py index 9c236dd..755607b 100644 --- a/cumin/cli.py +++ b/cumin/cli.py @@ -110,7 +110,7 @@ def parse_args(argv=None): '[optional]')) parser.add_argument('--dry-run', action='store_true', help='Do not execute any command, just return the list of matching hosts and exit.') - parser.add_argument('--version', action='store_true', help='Print current version and exit.') + parser.add_argument('--version', action='version', version='%(prog)s {version}'.format(version=cumin.__version__)) parser.add_argument('-d', '--debug', action='store_true', help='Set log level to DEBUG.') parser.add_argument('--trace', action='store_true', help='Set log level to TRACE, a custom logging level intended for development debugging.') @@ -353,10 +353,6 @@ def main(argv=None): # Setup try: args = parse_args(argv) - if args.version: - tqdm.write('cumin {version}'.format(version=cumin.__version__)) - return 0 - user = get_running_user() config = cumin.Config(args.config) setup_logging(config['log_file'], debug=args.debug, trace=args.trace) diff --git a/cumin/tests/fixtures/backends/grammars/openstack_invalid.txt b/cumin/tests/fixtures/backends/grammars/openstack_invalid.txt new file mode 100644 index 0000000..2c6cc72 --- /dev/null +++ b/cumin/tests/fixtures/backends/grammars/openstack_invalid.txt @@ -0,0 +1,6 @@ +# Invalid grammars +* key:value +key:value key +key:value :value +"key":value +key%:value diff --git a/cumin/tests/fixtures/backends/grammars/openstack_valid.txt b/cumin/tests/fixtures/backends/grammars/openstack_valid.txt new file mode 100644 index 0000000..f4b3d01 --- /dev/null +++ b/cumin/tests/fixtures/backends/grammars/openstack_valid.txt @@ -0,0 +1,10 @@ +# Valid grammars +* +name:host1 +name:host10.* +name:"^host10[1-9]\.domain$" +name:'^host10[1-9]\.domain$' +project:project_name +project:"Project Name" +project:project_name name:host1 +project:"Project Name" name:host1 diff --git a/cumin/tests/integration/test_cli.py b/cumin/tests/integration/test_cli.py index d2c0e84..3ae90cd 100644 --- a/cumin/tests/integration/test_cli.py +++ b/cumin/tests/integration/test_cli.py @@ -9,7 +9,7 @@ import pytest -from cumin import cli +from cumin import __version__, cli # Set environment variables _ENV = {'USER': 'root', 'SUDO_USER': 'user'} @@ -376,3 +376,17 @@ def test_timeout(self, capsys): assert _EXPECTED_LINES['all_failure'] in err, _EXPECTED_LINES['all_failure'] assert _EXPECTED_LINES['failed'] not in err, _EXPECTED_LINES['failed'] assert rc == 2 + + def test_version(self, capsys): # pylint: disable=no-self-use + """Calling --version should return the version and exit.""" + with pytest.raises(SystemExit) as e: + cli.main(argv=['--version']) + + out, err = capsys.readouterr() + sys.stdout.write(out) + sys.stderr.write(err) + assert e.type == SystemExit + assert e.value.code == 0 + assert out == '' + assert len(err.splitlines()) == 1 + assert __version__ in err diff --git a/cumin/tests/unit/backends/test_openstack.py b/cumin/tests/unit/backends/test_openstack.py new file mode 100644 index 0000000..aff04e0 --- /dev/null +++ b/cumin/tests/unit/backends/test_openstack.py @@ -0,0 +1,129 @@ +"""OpenStack backend tests.""" +from collections import namedtuple + +import mock + +from ClusterShell.NodeSet import NodeSet + +from cumin.backends import BaseQuery, openstack + + +Project = namedtuple('Project', ['name']) +Server = namedtuple('Server', ['name']) + + +def test_openstack_query_class(): + """An instance of query_class should be an instance of BaseQuery.""" + query = openstack.query_class({}) + assert isinstance(query, BaseQuery) + + +def test_openstack_query_class_init(): + """An instance of OpenStackQuery should be an instance of BaseQuery.""" + config = {'key': 'value'} + query = openstack.OpenStackQuery(config) + assert isinstance(query, BaseQuery) + assert query.config == config + + +def test_all_selection(): + """A selection for all hosts is properly parsed and interpreted.""" + parsed = openstack.grammar().parseString('*', parseAll=True) + assert parsed[0].asDict() == {'all': '*'} + + +def test_key_value_token(): + """A token is properly parsed and interpreted.""" + parsed = openstack.grammar().parseString('project:project_name', parseAll=True) + assert parsed[0].asDict() == {'key': 'project', 'value': 'project_name'} + + +def test_key_value_tokens(): + """Multiple tokens are properly parsed and interpreted.""" + parsed = openstack.grammar().parseString('project:project_name name:hostname', parseAll=True) + assert parsed[0].asDict() == {'key': 'project', 'value': 'project_name'} + assert parsed[1].asDict() == {'key': 'name', 'value': 'hostname'} + + +@mock.patch('cumin.backends.openstack.nova_client.Client') +@mock.patch('cumin.backends.openstack.keystone_client.Client') +@mock.patch('cumin.backends.openstack.keystone_session.Session') +@mock.patch('cumin.backends.openstack.keystone_identity.Password') +class TestOpenStackQuery(object): + """OpenStack backend query test class.""" + + def setup_method(self, _): + """Set an instance of OpenStackQuery for each test.""" + self.config = {'openstack': {}} # pylint: disable=attribute-defined-outside-init + self.query = openstack.OpenStackQuery(self.config) # pylint: disable=attribute-defined-outside-init + + def test_execute_all(self, keystone_identity, keystone_session, keystone_client, nova_client): + """Calling execute() with a query that select all hosts should return the list of all hosts.""" + keystone_client.return_value.projects.list.return_value = [Project('project1'), Project('project2')] + nova_client.return_value.servers.list.side_effect = [ + [Server('host1'), Server('host2')], [Server('host1'), Server('host2')]] + + hosts = self.query.execute('*') + assert hosts == NodeSet('host[1-2].project[1-2]') + + assert keystone_identity.call_count == 3 + assert keystone_session.call_count == 3 + keystone_client.assert_called_once_with(session=keystone_session(), timeout=10) + assert nova_client.call_args_list == [ + mock.call('2', endpoint_type='public', session=keystone_session(), timeout=10), + mock.call('2', endpoint_type='public', session=keystone_session(), timeout=10)] + assert nova_client().servers.list.call_args_list == [ + mock.call(search_opts={'vm_state': 'ACTIVE', 'status': 'ACTIVE'})] * 2 + + def test_execute_project(self, keystone_identity, keystone_session, keystone_client, nova_client): + """Calling execute() with a query that select all hosts in a project should return the list of hosts.""" + nova_client.return_value.servers.list.return_value = [Server('host1'), Server('host2')] + + hosts = self.query.execute('project:project1') + assert hosts == NodeSet('host[1-2].project1') + + assert keystone_identity.call_count == 1 + assert keystone_session.call_count == 1 + keystone_client.assert_not_called() + nova_client.assert_called_once_with('2', endpoint_type='public', session=keystone_session(), timeout=10) + nova_client().servers.list.assert_called_once_with(search_opts={'vm_state': 'ACTIVE', 'status': 'ACTIVE'}) + + def test_execute_project_name(self, keystone_identity, keystone_session, keystone_client, nova_client): + """Calling execute() with a query that select hosts matching a name in a project should return only those.""" + nova_client.return_value.servers.list.return_value = [Server('host1'), Server('host2')] + + hosts = self.query.execute('project:project1 name:host') + assert hosts == NodeSet('host[1-2].project1') + + assert keystone_identity.call_count == 1 + assert keystone_session.call_count == 1 + keystone_client.assert_not_called() + nova_client.assert_called_once_with('2', endpoint_type='public', session=keystone_session(), timeout=10) + nova_client().servers.list.assert_called_once_with( + search_opts={'vm_state': 'ACTIVE', 'status': 'ACTIVE', 'name': 'host'}) + + def test_execute_project_domain(self, keystone_identity, keystone_session, keystone_client, nova_client): + """When the domain suffix is configured, it should append it to all hosts.""" + nova_client.return_value.servers.list.return_value = [Server('host1'), Server('host2')] + self.config['openstack']['domain_suffix'] = 'servers.local' + query = openstack.OpenStackQuery(self.config) + + hosts = query.execute('project:project1') + assert hosts == NodeSet('host[1-2].project1.servers.local') + + assert keystone_identity.call_count == 1 + assert keystone_session.call_count == 1 + keystone_client.assert_not_called() + + def test_execute_project_dot_domain(self, keystone_identity, keystone_session, keystone_client, nova_client): + """When the domain suffix is configured with a dot, it should append it to all hosts without the dot.""" + nova_client.return_value.servers.list.return_value = [Server('host1'), Server('host2')] + self.config['openstack']['domain_suffix'] = '.servers.local' + query = openstack.OpenStackQuery(self.config) + + hosts = query.execute('project:project1') + assert hosts == NodeSet('host[1-2].project1.servers.local') + + assert keystone_identity.call_count == 1 + assert keystone_session.call_count == 1 + keystone_client.assert_not_called() diff --git a/cumin/tests/unit/transports/test_clustershell.py b/cumin/tests/unit/transports/test_clustershell.py index 39c6112..f889e2d 100644 --- a/cumin/tests/unit/transports/test_clustershell.py +++ b/cumin/tests/unit/transports/test_clustershell.py @@ -95,9 +95,10 @@ def test_execute_custom_handler(self): assert kwargs['handler'] == self.worker._handler_instance def test_execute_no_commands(self): - """Calling execute() without commands should raise WorkerError.""" - with pytest.raises(WorkerError, match=r'commands must be a non-empty list'): - self.worker.commands = [] + """Calling execute() without commands should return immediately.""" + self.worker.handler = ConcreteBaseEventHandler + self.worker.commands = None + assert self.worker.execute() is None assert not self.worker.task.shell.called def test_execute_one_command_no_mode(self): @@ -106,6 +107,13 @@ def test_execute_one_command_no_mode(self): with pytest.raises(RuntimeError, match=r'An EventHandler is mandatory\.'): self.worker.execute() + def test_execute_no_target_hosts(self): + """Calling execute() with a target without hosts should return immediately.""" + self.target.hosts = [] + self.worker.handler = ConcreteBaseEventHandler + assert self.worker.execute() is None + assert not self.worker.task.shell.called + def test_execute_wrong_mode(self): """Calling execute() without setting the mode with multiple commands should raise RuntimeError.""" with pytest.raises(RuntimeError, match=r'An EventHandler is mandatory\.'): @@ -273,7 +281,7 @@ def test_ev_read_single_host(self, tqdm): self.worker.current_node = self.target.hosts[0] self.worker.current_msg = output self.handler.ev_read(self.worker) - tqdm.write.assert_has_calls([mock.call(output)]) + assert tqdm.write.call_args[0][0] == output def test_ev_timeout(self): """Calling ev_timeout() should increase the counters for the timed out hosts.""" diff --git a/cumin/tests/unit/transports/test_init.py b/cumin/tests/unit/transports/test_init.py index 8f75535..c40dfdc 100644 --- a/cumin/tests/unit/transports/test_init.py +++ b/cumin/tests/unit/transports/test_init.py @@ -427,6 +427,8 @@ def test_commands_setter(self): assert self.worker._commands == self.commands self.worker.commands = None assert self.worker._commands is None + self.worker.commands = [] + assert self.worker._commands == [] self.worker.commands = ['command1', 'command2'] assert self.worker._commands == self.commands diff --git a/cumin/tests/vulture_whitelist.py b/cumin/tests/vulture_whitelist.py index 5613903..3e0feb0 100644 --- a/cumin/tests/vulture_whitelist.py +++ b/cumin/tests/vulture_whitelist.py @@ -18,6 +18,9 @@ def __getattr__(self, _): whitelist_transports_clustershell = Whitelist() whitelist_transports_clustershell.BaseEventHandler.kwargs +whitelist_backends_openstack = Whitelist() +whitelist_backends_openstack.TestOpenStackQuery.test_execute_all.nova_client.return_value.servers.list.side_effect + whitelist_tests_integration_conftest = Whitelist() whitelist_tests_integration_conftest.pytest_cmdline_preparse whitelist_tests_integration_conftest.pytest_runtest_makereport diff --git a/cumin/transports/__init__.py b/cumin/transports/__init__.py index 50d3a15..a51347b 100644 --- a/cumin/transports/__init__.py +++ b/cumin/transports/__init__.py @@ -347,7 +347,7 @@ def commands(self, value): self._commands = value return - validate_list('commands', value) + validate_list('commands', value, allow_empty=True) commands = [] for command in value: if isinstance(command, Command): diff --git a/cumin/transports/clustershell.py b/cumin/transports/clustershell.py index c2b5c30..aec3ca7 100644 --- a/cumin/transports/clustershell.py +++ b/cumin/transports/clustershell.py @@ -38,6 +38,10 @@ def execute(self): self.logger.warning('No commands provided') return + if not self.target.hosts: + self.logger.warning('No target hosts provided') + return + if self.handler is None: raise RuntimeError('An EventHandler is mandatory.') @@ -238,7 +242,7 @@ def ev_read(self, worker): if self.deduplicate_output: return - tqdm.write(worker.current_msg) + tqdm.write(worker.current_msg, file=sys.stdout) def ev_timeout(self, worker): """Worker has timed out. diff --git a/doc/examples/aliases.yaml b/doc/examples/aliases.yaml index f7d2a23..4ea5a09 100644 --- a/doc/examples/aliases.yaml +++ b/doc/examples/aliases.yaml @@ -1,3 +1,4 @@ alias_direct: D{host1 or host2} alias_puppetdb: P{R:Class = My::Class} -alias_all: A:alias_direct and (A:alias_puppetdb and not D{host3}) +alias_openstack: O{project:project_name} +alias_complex: A:alias_direct and (A:alias_puppetdb and not D{host3}) diff --git a/doc/examples/config.yaml b/doc/examples/config.yaml index 47334cd..d7ed450 100644 --- a/doc/examples/config.yaml +++ b/doc/examples/config.yaml @@ -8,13 +8,22 @@ default_backend: direct environment: ENV_VARIABLE: env_value -# Backend-specific configuration +# Backend-specific configurations [optional] puppetdb: host: puppetdb.local port: 443 urllib3_disable_warnings: # List of Python urllib3 exceptions to ignore - SubjectAltNameWarning +openstack: + auth_url: http://keystone.local:5000 + username: observer # Keystone API user's username + password: observer_password # Keystone API user's password + domain_suffix: openstack.local # OpenStack managed domain, to be added to all hostnames + nova_api_version: 2.12 + timeout: 2 # Used for both Keystone and Nova API calls + + # Transport-specific configuration clustershell: ssh_options: # SSH options passed to ClusterShell [optional] diff --git a/setup.py b/setup.py index ca9a2b9..5816235 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,10 @@ install_requires = [ 'clustershell==1.7.3', 'colorama>=0.3.2', + 'keystoneauth1>=2.4.1', 'pyparsing==2.1.10', + 'python-keystoneclient>=2.3.1', + 'python-novaclient>=3.3.1', 'pyyaml>=3.11', 'requests>=2.12.0', 'tqdm>=4.11.2', @@ -52,7 +55,7 @@ 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', ], - data_files=[('doc/cumin/examples/', ['doc/examples/config.yaml', 'doc/examples/aliases.yaml'])], + data_files=[('share/doc/cumin/examples/', ['doc/examples/config.yaml', 'doc/examples/aliases.yaml'])], description='Automation and orchestration framework written in Python', entry_points={ 'console_scripts': [