Migration to Python 3

* Migrate Cumin to Python 3, dropping support of Python 2
* Besides the usual Py2 -> Py3 conversions, the main changes are:
  * Add nodeset() and nodeset_fromlist() functions to instantiate
    ClusterShell's NodeSet objects with the resolver set to
    RESOLVER_NOGROUP, due to:
    cea-hpc/clustershell#368
  * Adapt to new ClusterShell's API signatures
  * Use threading.Lock() calls as context managers for the 'with'
    statement
  * Use Colorama autoreset feature, simplifying its related calls

Change-Id: I676b3497a942db827fb42d2caa125551007a4d01

.gitignore

@@ -4,6 +4,7 @@
 /.coverage
 /.coverage-integration-clustershell
 /.eggs/
+/.pytest_cache/
 /.tox/
 /.venv/
 /dist/

.travis.yml

@@ -1,9 +1,13 @@
 language: python
-python: 2.7
+python:
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7-dev"
 before_install:
   - pip install --upgrade setuptools
 install:
-    - pip install --upgrade coveralls codecov tox
+  - pip install --upgrade coveralls codecov tox
 script: tox -e unit
 after_success:
   - coveralls

README.rst

@@ -11,8 +11,9 @@ selected, and can provide multiple execution strategies. The executed commands o
 easy-to-read result.
 
 
-It can be used both via its command line interface (CLI) `cumin` and as a Python 2 library. Python 3 support will be
-added soon, as the last dependency that was Python 2 only added support for Python 3 recently.
+It can be used both via its command line interface (CLI) `cumin` and as a Python 3 only library.
+Cumin was Python 2 only before the 3.0.0 release, due to ClusterShell not yet being Python 3 compatible.
+
 
 |Cumin GIF|
 

cumin/__init__.py

@@ -6,6 +6,8 @@
 
 import yaml
 
+from ClusterShell.NodeSet import NodeSet, RESOLVER_NOGROUP
+
 
 try:
     __version__ = get_distribution(__name__).version
@@ -28,8 +30,8 @@ class CuminError(Exception):
 # Fail if the custom logging slot is already in use with a different name or
 # Access to a private property of logging was preferred over matching the default string returned by
 # logging.getLevelName() for unused custom slots.
-if (LOGGING_TRACE_LEVEL_NUMBER in logging._levelNames and  # pylint: disable=protected-access
-        LOGGING_TRACE_LEVEL_NAME not in logging._levelNames):  # pylint: disable=protected-access
+if (LOGGING_TRACE_LEVEL_NUMBER in logging._levelToName and  # pylint: disable=protected-access
+        LOGGING_TRACE_LEVEL_NAME not in logging._nameToLevel):  # pylint: disable=protected-access
     raise CuminError("Unable to set custom logging for trace, logging level {level} is alredy set for '{name}'.".format(
         level=LOGGING_TRACE_LEVEL_NUMBER, name=logging.getLevelName(LOGGING_TRACE_LEVEL_NUMBER)))
 
@@ -46,7 +48,7 @@ def trace(self, msg, *args, **kwargs):
 
 
 # Install the trace method and it's logging level if not already present
-if LOGGING_TRACE_LEVEL_NAME not in logging._levelNames:  # pylint: disable=protected-access
+if LOGGING_TRACE_LEVEL_NAME not in logging._nameToLevel:  # pylint: disable=protected-access
     logging.addLevelName(LOGGING_TRACE_LEVEL_NUMBER, LOGGING_TRACE_LEVEL_NAME)
 if not hasattr(logging.Logger, 'trace'):
     logging.Logger.trace = trace
@@ -109,3 +111,33 @@ def parse_config(config_file):
         config = {}
 
     return config
+
+
+def nodeset(nodes=None):
+    """Instantiate a ClusterShell NodeSet with the resolver defaulting to :py:const:`RESOLVER_NOGROUP`.
+
+    This allow to avoid any conflict with Cumin grammars.
+
+    Returns:
+        ClusterShell.NodeSet.NodeSet: the instantiated NodeSet.
+
+    See Also:
+        https://github.com/cea-hpc/clustershell/issues/368
+
+    """
+    return NodeSet(nodes=nodes, resolver=RESOLVER_NOGROUP)
+
+
+def nodeset_fromlist(nodelist):
+    """Instantiate a ClusterShell NodeSet from a list with the resolver defaulting to :py:const:`RESOLVER_NOGROUP`.
+
+    This allow to avoid any conflict with Cumin grammars.
+
+    Returns:
+        ClusterShell.NodeSet.NodeSet: the instantiated NodeSet.
+
+    See Also:
+        https://github.com/cea-hpc/clustershell/issues/368
+
+    """
+    return NodeSet.fromlist(nodelist, resolver=RESOLVER_NOGROUP)

cumin/backends/__init__.py

@@ -6,23 +6,19 @@
 
 import pyparsing
 
-from ClusterShell.NodeSet import NodeSet
-
-from cumin import CuminError
+from cumin import CuminError, nodeset
 
 
 class InvalidQueryError(CuminError):
     """Custom exception class for invalid queries."""
 
 
-class BaseQuery(object):
+class BaseQuery(object, metaclass=ABCMeta):
     """Query abstract class.
 
     All backends query classes must inherit, directly or indirectly, from this one.
     """
 
-    __metaclass__ = ABCMeta
-
     grammar = pyparsing.NoMatch()  # This grammar will never match.
     """:py:class:`pyparsing.ParserElement`: derived classes must define their own pyparsing grammar and set this class
     attribute accordingly."""
@@ -96,7 +92,7 @@ def __init__(self, config):
         :Parameters:
             according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
         """
-        super(BaseQueryAggregator, self).__init__(config)
+        super().__init__(config)
 
         self.stack = None
         self.stack_pointer = None
@@ -109,7 +105,7 @@ def _build(self, query_string):
         """
         self.stack = self._get_stack_element()
         self.stack_pointer = self.stack
-        super(BaseQueryAggregator, self)._build(query_string)
+        super()._build(query_string)
         self.logger.trace('Query stack: %s', self.stack)
 
     def _execute(self):
@@ -118,8 +114,8 @@ def _execute(self):
         :Parameters:
             according to parent :py:meth:`cumin.backends.BaseQuery._execute`.
         """
-        hosts = NodeSet()
-        self._loop_stack(hosts, self.stack)  # The hosts nodeset is updated in place while looping the stack
+        hosts = nodeset()
+        self._loop_stack(hosts, self.stack)  # The hosts NodeSet is updated in place while looping the stack
         self.logger.debug('Found %d hosts', len(hosts))
 
         return hosts
@@ -162,7 +158,7 @@ def _loop_stack(self, hosts, stack_element):
             stack_element (dict): the stack element to iterate.
         """
         if stack_element['hosts'] is None:
-            element_hosts = NodeSet()
+            element_hosts = nodeset()
             for child in stack_element['children']:
                 self._loop_stack(element_hosts, child)
         else:
@@ -175,9 +171,9 @@ def _aggregate_hosts(self, hosts, element_hosts, bool_operator):
 
         Arguments:
             hosts (ClusterShell.NodeSet.NodeSet): the hosts to update with the results in ``element_hosts`` according
-                to the bool_operator. This object is updated in place by reference.
-            element_hosts (ClusterShell.NodeSet.NodeSet): the additional hosts to aggregate to the results based on
-                the ``bool_operator``.
+                to the ``bool_operator``. This object is updated in place by reference.
+            element_hosts (ClusterShell.NodeSet.NodeSet): the additional hosts to aggregate to the results based on the
+                ``bool_operator``.
             bool_operator (str, None): the boolean operator to apply while aggregating the two NodeSet. It must be
                 :py:data:`None` when adding the first hosts.
         """

cumin/backends/direct.py

@@ -1,8 +1,7 @@
 """Direct backend."""
 import pyparsing as pp
 
-from ClusterShell.NodeSet import NodeSet
-
+from cumin import nodeset_fromlist
 from cumin.backends import BaseQueryAggregator, InvalidQueryError
 
 
@@ -78,7 +77,7 @@ def _parse_token(self, token):
 
         if 'hosts' in token_dict:
             element = self._get_stack_element()
-            element['hosts'] = NodeSet.fromlist(token_dict['hosts'])
+            element['hosts'] = nodeset_fromlist(token_dict['hosts'])
             if 'bool' in token_dict:
                 element['bool'] = token_dict['bool']
             self.stack_pointer['children'].append(element)

cumin/backends/openstack.py

@@ -1,12 +1,12 @@
 """OpenStack backend."""
 import pyparsing as pp
 
-from ClusterShell.NodeSet import NodeSet
 from keystoneauth1 import session as keystone_session
 from keystoneauth1.identity import v3 as keystone_identity
 from keystoneclient.v3 import client as keystone_client
 from novaclient import client as nova_client
 
+from cumin import nodeset, nodeset_fromlist
 from cumin.backends import BaseQuery, InvalidQueryError
 
 
@@ -124,7 +124,7 @@ def __init__(self, config):
             according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
 
         """
-        super(OpenStackQuery, self).__init__(config)
+        super().__init__(config)
         self.openstack_config = self.config.get('openstack', {})
         self.search_project = None
         self.search_params = self._get_default_search_params()
@@ -153,7 +153,7 @@ def _build(self, query_string):
 
         """
         self.search_params = self._get_default_search_params()
-        super(OpenStackQuery, self)._build(query_string)
+        super()._build(query_string)
 
     def _execute(self):
         """Concrete implementation of parent abstract method.
@@ -166,7 +166,7 @@ def _execute(self):
 
         """
         if self.search_project is None:
-            hosts = NodeSet()
+            hosts = nodeset()
             for project in self._get_projects():
                 hosts |= self._get_project_hosts(project)
         else:
@@ -232,7 +232,7 @@ def _get_project_hosts(self, project):
             else:
                 domain = domain_suffix
 
-        return NodeSet.fromlist('{host}.{project}{domain}'.format(host=server.name, project=project, domain=domain)
+        return nodeset_fromlist('{host}.{project}{domain}'.format(host=server.name, project=project, domain=domain)
                                 for server in client.servers.list(search_opts=self.search_params))
 
 

cumin/backends/puppetdb.py

@@ -6,9 +6,9 @@
 import pyparsing as pp
 import requests
 
-from ClusterShell.NodeSet import NodeSet
 from requests.packages import urllib3
 
+from cumin import nodeset, nodeset_fromlist
 from cumin.backends import BaseQuery, InvalidQueryError
 
 
@@ -158,7 +158,7 @@ def __init__(self, config):
             according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
 
         """
-        super(PuppetDBQuery, self).__init__(config)
+        super().__init__(config)
         self.grouped_tokens = None
         self.current_group = self.grouped_tokens
         self._endpoint = None
@@ -237,7 +237,7 @@ def _build(self, query_string):
         """
         self.grouped_tokens = PuppetDBQuery._get_grouped_tokens()
         self.current_group = self.grouped_tokens
-        super(PuppetDBQuery, self)._build(query_string)
+        super()._build(query_string)
         self.logger.trace('Query tokens: %s', self.grouped_tokens)
 
     def _execute(self):
@@ -252,7 +252,7 @@ def _execute(self):
         """
         query = self._get_query_string(group=self.grouped_tokens).format(host_key=self.hosts_keys[self.endpoint])
         hosts = self._api_call(query)
-        unique_hosts = NodeSet.fromlist([host[self.hosts_keys[self.endpoint]] for host in hosts])
+        unique_hosts = nodeset_fromlist([host[self.hosts_keys[self.endpoint]] for host in hosts])
         self.logger.debug("Queried puppetdb for '%s', got '%d' results.", query, len(unique_hosts))
 
         return unique_hosts
@@ -342,7 +342,7 @@ def _parse_token(self, token):
             self._add_bool(token_dict['bool'])
 
         elif 'hosts' in token_dict:
-            token_dict['hosts'] = NodeSet(token_dict['hosts'])
+            token_dict['hosts'] = nodeset(token_dict['hosts'])
             self._add_hosts(**token_dict)
 
         elif 'category' in token_dict:

cumin/cli.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 """Cumin CLI entry point."""
 import argparse
 import code
@@ -13,7 +13,6 @@
 
 import colorama
 
-from ClusterShell.NodeSet import NodeSet
 from tqdm import tqdm
 
 import cumin
@@ -45,7 +44,7 @@
 = Example usage:
 for nodes, output in worker.get_results():
     print(nodes)
-    print(output)
+    print(output.message().decode('utf-8'))
     print('-----')
 """
 """str: The message to print when entering the intractive REPL mode."""
@@ -86,7 +85,7 @@ def get_parser():
                               '-p/--success-percentage is reached. In async mode, execute on each host independently '
                               'from each other, the list of commands, aborting the execution on any given host at the '
                               'first command that fails.'))
-    parser.add_argument('-p', '--success-percentage', type=int, choices=xrange(101), metavar='PCT', default=100,
+    parser.add_argument('-p', '--success-percentage', type=int, choices=range(101), metavar='PCT', default=100,
                         help=(('Percentage threshold to consider an execution unit successful. Required in sync mode, '
                                'optional in async mode when -b/--batch-size is used. Accepted values are integers '
                                'in the range 0-100. [default: 100]')))
@@ -186,7 +185,7 @@ def setup_logging(filename, debug=False, trace=False):
     """
     file_path = os.path.dirname(filename)
     if not os.path.exists(file_path):
-        os.makedirs(file_path, 0770)
+        os.makedirs(file_path, 0o770)
 
     log_formatter = logging.Formatter(fmt='%(asctime)s [%(levelname)s %(process)s %(name)s.%(funcName)s] %(message)s')
     log_handler = RotatingFileHandler(filename, maxBytes=(5 * (1024**2)), backupCount=30)
@@ -207,7 +206,7 @@ def setup_logging(filename, debug=False, trace=False):
 def sigint_handler(*args):  # pylint: disable=unused-argument
     """Signal handler for Ctrl+c / SIGINT, raises KeyboardInterruptError.
 
-    Arguments (as defined in https://docs.python.org/2/library/signal.html):
+    Arguments (as defined in https://docs.python.org/3/library/signal.html):
         signum: the signal number
         frame: the current stack frame
     """
@@ -223,7 +222,7 @@ def sigint_handler(*args):  # pylint: disable=unused-argument
     # for i in xrange(10):
     #     stderr('Ctrl+c pressed, sure to quit [y/n]?\n')
     #     try:
-    #         answer = raw_input('\n')
+    #         answer = input('\n')  # nosec
     #     except RuntimeError:
     #         # Can't re-enter readline when already waiting for input in get_hosts(). Assuming 'y' as answer
     #         stderr('Ctrl+c pressed while waiting for answer. Aborting')
@@ -270,7 +269,7 @@ def get_hosts(args, config):
         return hosts
 
     stderr('{num} hosts will be targeted:'.format(num=len(hosts)))
-    stderr('{color}{hosts}'.format(color=colorama.Fore.CYAN, hosts=NodeSet.fromlist(hosts)))
+    stderr('{color}{hosts}'.format(color=colorama.Fore.CYAN, hosts=cumin.nodeset_fromlist(hosts)))
 
     if args.dry_run:
         stderr('DRY-RUN mode enabled, aborting')
@@ -283,9 +282,9 @@ def get_hosts(args, config):
         stderr(message)
         raise cumin.CuminError(message)
 
-    for i in xrange(10):
+    for i in range(10):
         stderr('Confirm to continue [y/n]?', end=' ')
-        answer = raw_input()
+        answer = input()  # nosec
         if not answer:
             continue
 
@@ -349,7 +348,7 @@ def run(args, config):
                        for command in args.commands]
     worker.timeout = args.global_timeout
     worker.handler = args.mode
-    worker.success_threshold = args.success_percentage / float(100)
+    worker.success_threshold = args.success_percentage / 100
     exit_code = worker.execute()
 
     if args.interactive: