utils.py cleanup. (errbotio#991)

* utils.py cleanup.

I just applied a couple principles:

- This should not be an API of Errbot
- If not used in Errbot, it is removed
- If is it used only once in Errbot, it is moved to the spot where it is
used
- If it is used a couple times in Errbot and doesn't have a natural spot
within the source tree, it stays.

* linting

Author: gbin

docs/user_guide/plugin_development/exceptions.rst

@@ -43,15 +43,15 @@ When handling exceptions, follow these steps:
     not re-raising it, we prevent that automatic "Computer says nooo.
     ..." message from being sent
 
-Also, note that there is a ``utils.ValidationException`` class which you
+Also, note that there is a ``errbot.ValidationException`` class which you
 can use inside your helper methods to raise meaningful errors and handle
 them accordingly.
 
 Here's an example:
 
 .. code-block:: python
 
-    from errbot import BotPlugin, arg_botcmd, utils
+    from errbot import BotPlugin, arg_botcmd, ValidationException
 
     class FooBot(BotPlugin):
         """An example bot"""
@@ -61,7 +61,7 @@ Here's an example:
             """Add your first name if it doesn't contain any digits"""
             try:
                 FooBot.validate_first_name(first_name)
-            except utils.ValidationException as exc:
+            except ValidationException as exc:
                 self.log.exception(
                     'first_name=%s contained a digit' % first_name
                 )
@@ -74,7 +74,7 @@ Here's an example:
         @staticmethod
         def validate_first_name(first_name):
             if any(char.isdigit() for char in first_name):
-                raise utils.ValidationException(
+                raise ValidationException(
                     "first_name=%s contained a digit" % first_name
                 )
 

errbot/__init__.py

@@ -11,7 +11,7 @@
 
 from .core_plugins.wsview import bottle_app, WebView
 from .backends.base import Message, ONLINE, OFFLINE, AWAY, DND  # noqa
-from .botplugin import BotPlugin, SeparatorArgParser, ShlexArgParser, CommandError, Command  # noqa
+from .botplugin import BotPlugin, SeparatorArgParser, ShlexArgParser, CommandError, Command, ValidationException  # noqa
 from .flow import FlowRoot, BotFlow, Flow, FLOW_END
 from .core_plugins.wsview import route, view  # noqa
 from . import core

errbot/botplugin.py

@@ -5,13 +5,45 @@
 from typing import Tuple, Callable, Mapping, Sequence
 from io import IOBase
 
-from .utils import recurse_check_structure
 from .storage import StoreMixin, StoreNotOpenError
 from errbot.backends.base import Message, Presence, Stream, Room, Identifier, ONLINE, Card
 
 log = logging.getLogger(__name__)
 
 
+class ValidationException(Exception):
+    pass
+
+
+def _recurse_check_plugin_configuration(sample, to_check):
+    sample_type = type(sample)
+    to_check_type = type(to_check)
+
+    # Skip this check if the sample is None because it will always be something
+    # other than NoneType when changed from the default. Raising ValidationException
+    # would make no sense then because it would defeat the whole purpose of having
+    # that key in the sample when it could only ever be None.
+    if sample is not None and sample_type != to_check_type:
+        raise ValidationException(
+            '%s [%s] is not the same type as %s [%s]' % (sample, sample_type, to_check, to_check_type))
+
+    if sample_type in (list, tuple):
+        for element in to_check:
+            _recurse_check_plugin_configuration(sample[0], element)
+        return
+
+    if sample_type == dict:
+        for key in sample:
+            if key not in to_check:
+                raise ValidationException("%s doesn't contain the key %s" % (to_check, key))
+        for key in to_check:
+            if key not in sample:
+                raise ValidationException("%s contains an unknown key %s" % (to_check, key))
+        for key in sample:
+            _recurse_check_plugin_configuration(sample[key], to_check[key])
+        return
+
+
 class CommandError(Exception):
     """
     Use this class to report an error condition from your commands, the command
@@ -312,7 +344,7 @@ def check_configuration(self, configuration: Mapping) -> None:
 
         :param configuration: the configuration to be checked.
         """
-        recurse_check_structure(self.get_configuration_template(), configuration)  # default behavior
+        _recurse_check_plugin_configuration(self.get_configuration_template(), configuration)  # default behavior
 
     def configure(self, configuration: Mapping) -> None:
         """

errbot/core.py

@@ -30,7 +30,7 @@
 from .storage import StoreMixin
 from .streaming import Tee
 from .templating import tenv
-from .utils import split_string_after, get_class_that_defined_method
+from .utils import split_string_after
 
 log = logging.getLogger(__name__)
 
@@ -653,8 +653,15 @@ def get_doc(self, command):
         pat = re.compile(r'!({})'.format('|'.join(ununderscore_keys)))
         return re.sub(pat, self.prefix + '\1', command.__doc__)
 
+    @staticmethod
+    def get_plugin_class_from_method(meth):
+        for cls in inspect.getmro(type(meth.__self__)):
+            if meth.__name__ in cls.__dict__:
+                return cls
+        return None
+
     def get_command_classes(self):
-        return (get_class_that_defined_method(command)
+        return (self.get_plugin_class_from_method(command)
                 for command in self.all_commands.values())
 
     def shutdown(self):

errbot/core_plugins/help.py

@@ -2,7 +2,6 @@
 
 from errbot import BotPlugin, botcmd
 from errbot.version import VERSION
-from errbot.utils import get_class_that_defined_method
 
 
 class Help(BotPlugin):
@@ -29,7 +28,7 @@ def apropos(self, mess, args):
 
         cls_commands = {}
         for (name, command) in self._bot.all_commands.items():
-            cls = get_class_that_defined_method(command)
+            cls = self._bot.get_plugin_class_from_method(command)
             cls = str.__module__ + '.' + cls.__name__  # makes the fuul qualified name
             commands = cls_commands.get(cls, [])
             if not self.bot_config.HIDE_RESTRICTED_COMMANDS or self._bot.check_command_access(mess, name)[0]:
@@ -79,7 +78,7 @@ def get_name(named):
         if not args:
             cls_commands = {}
             for (name, command) in self._bot.all_commands.items():
-                cls = get_class_that_defined_method(command)
+                cls = self._bot.get_plugin_class_from_method(command)
                 commands = cls_commands.get(cls, [])
                 if not self.bot_config.HIDE_RESTRICTED_COMMANDS or may_access_command(mess, name):
                     commands.append((name, command))
@@ -107,7 +106,7 @@ def get_name(named):
             commands = [
                 (name, command) for (name, command)
                 in self._bot.all_commands.items() if
-                get_name(get_class_that_defined_method(command)) == args]
+                get_name(self._bot.get_plugin_class_from_method(command)) == args]
 
             description = '\n**{name}**\n\n*{doc}*\n\n'.format(
                 name=cls.__name__,

errbot/core_plugins/utils.py

@@ -1,7 +1,10 @@
 from os import path
 
 from errbot import BotPlugin, botcmd
-from errbot.utils import tail
+
+
+def tail(f, window=20):
+    return ''.join(f.readlines()[-window:])
 
 
 class Utils(BotPlugin):

errbot/plugin_manager.py

@@ -12,7 +12,7 @@
 
 from errbot.flow import BotFlow
 from .botplugin import BotPlugin
-from .utils import version2array, collect_roots, ensure_sys_path_contains
+from .utils import version2array, collect_roots
 from .templating import remove_plugin_templates_path, add_plugin_templates_path
 from .version import VERSION
 from yapsy.PluginManager import PluginManager
@@ -46,6 +46,19 @@ class PluginConfigurationException(PluginActivationException):
     pass
 
 
+def _ensure_sys_path_contains(paths):
+    """ Ensure that os.path contains paths
+       :param base_paths:
+            a list of base paths to walk from
+            elements can be a string or a list/tuple of strings
+    """
+    for entry in paths:
+        if isinstance(entry, (list, tuple)):
+            _ensure_sys_path_contains(entry)
+        elif entry is not None and entry not in sys.path:
+            sys.path.append(entry)
+
+
 def populate_doc(plugin):
     plugin_type = type(plugin.plugin_object)
     plugin_type.__errdoc__ = plugin_type.__doc__ if plugin_type.__doc__ else plugin.description
@@ -370,7 +383,7 @@ def update_plugin_places(self, path_list, extra_plugin_dir, autoinstall_deps=Tru
                 log.debug("Add %s to sys.path", entry)
                 sys.path.append(entry)
         # so plugins can relatively import their repos
-        ensure_sys_path_contains(repo_roots)
+        _ensure_sys_path_contains(repo_roots)
 
         errors = {}
         if autoinstall_deps:

errbot/repo_manager.py

@@ -16,7 +16,7 @@
 
 from errbot.plugin_manager import check_dependencies
 from errbot.storage import StoreMixin
-from .utils import which
+from .utils import ON_WINDOWS
 
 log = logging.getLogger(__name__)
 
@@ -61,6 +61,26 @@ def tokenizeJsonEntry(json_dict):
     return set(find_words.findall(' '.join((word.lower() for word in json_dict.values()))))
 
 
+def which(program):
+    if ON_WINDOWS:
+        program += '.exe'
+
+    def is_exe(file_path):
+        return os.path.isfile(file_path) and os.access(file_path, os.X_OK)
+
+    fpath, fname = os.path.split(program)
+    if fpath:
+        if is_exe(program):
+            return program
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            exe_file = os.path.join(path, program)
+            if is_exe(exe_file):
+                return exe_file
+
+    return None
+
+
 class BotRepoManager(StoreMixin):
     """
     Manages the repo list, git clones/updates or the repos.

errbot/streaming.py

@@ -1,16 +1,30 @@
 
 import os
 import io
+from itertools import starmap, repeat
 from threading import Thread
 from .backends.base import STREAM_WAITING_TO_START, STREAM_TRANSFER_IN_PROGRESS
 import logging
-from .utils import repeatfunc
 
 CHUNK_SIZE = 4096
 
 log = logging.getLogger(__name__)
 
 
+def repeatfunc(func, times=None, *args):  # from the itertools receipes
+    """Repeat calls to func with specified arguments.
+
+    Example:  repeatfunc(random.random)
+
+    :param args: params to the function to call.
+    :param times: number of times to repeat.
+    :param func:  the function to repeatedly call.
+    """
+    if times is None:
+        return starmap(func, repeat(args))
+    return starmap(func, repeat(args, times))
+
+
 class Tee(object):
     """ Tee implements a multi reader / single writer """
     def __init__(self, incoming_stream, clients):