diff --git a/mycroft/skills/event_scheduler.py b/mycroft/skills/event_scheduler.py index 38b3b0023634..1df98c1959f7 100644 --- a/mycroft/skills/event_scheduler.py +++ b/mycroft/skills/event_scheduler.py @@ -17,12 +17,14 @@ """ import json import time +from datetime import datetime, timedelta from threading import Thread, Lock from os.path import isfile, join, expanduser from mycroft.configuration import Configuration from mycroft.messagebus.message import Message from mycroft.util.log import LOG +from .mycroft_skill.event_container import EventContainer def repeat_time(sched_time, repeat): @@ -250,3 +252,176 @@ def shutdown(self): self.clear_empty() # Store all pending scheduled events self.store() + + +class EventSchedulerInterface: + """Interface for accessing the event scheduler over the message bus.""" + def __init__(self, name, sched_id=None, bus=None): + self.name = name + self.sched_id = sched_id + self.bus = bus + self.events = EventContainer(bus) + + self.scheduled_repeats = [] + + def set_bus(self, bus): + self.bus = bus + self.events.set_bus(bus) + + def set_id(self, sched_id): + self.sched_id = sched_id + + def _create_unique_name(self, name): + """Return a name unique to this skill using the format + [skill_id]:[name]. + + Arguments: + name: Name to use internally + + Returns: + str: name unique to this skill + """ + return str(self.sched_id) + ':' + (name or '') + + def _schedule_event(self, handler, when, data, name, repeat_interval=None): + """Underlying method for schedule_event and schedule_repeating_event. + + Takes scheduling information and sends it off on the message bus. + + Arguments: + handler: method to be called + when (datetime): time (in system timezone) for first + calling the handler, or None to + initially trigger seconds + from now + data (dict, optional): data to send when the handler is called + name (str, optional): reference name, must be unique + repeat_interval (float/int): time in seconds between calls + + """ + if isinstance(when, (int, float)) and when >= 0: + when = datetime.now() + timedelta(seconds=when) + if not name: + name = self.name + handler.__name__ + unique_name = self._create_unique_name(name) + if repeat_interval: + self.scheduled_repeats.append(name) # store "friendly name" + + data = data or {} + self.events.add(unique_name, handler, once=not repeat_interval) + event_data = {'time': time.mktime(when.timetuple()), + 'event': unique_name, + 'repeat': repeat_interval, + 'data': data} + self.bus.emit(Message('mycroft.scheduler.schedule_event', + data=event_data)) + + def schedule_event(self, handler, when, data=None, name=None): + """Schedule a single-shot event. + + Arguments: + handler: method to be called + when (datetime/int/float): datetime (in system timezone) or + number of seconds in the future when the + handler should be called + data (dict, optional): data to send when the handler is called + name (str, optional): reference name + NOTE: This will not warn or replace a + previously scheduled event of the same + name. + """ + self._schedule_event(handler, when, data, name) + + def schedule_repeating_event(self, handler, when, interval, + data=None, name=None): + """Schedule a repeating event. + + Arguments: + handler: method to be called + when (datetime): time (in system timezone) for first + calling the handler, or None to + initially trigger seconds + from now + interval (float/int): time in seconds between calls + data (dict, optional): data to send when the handler is called + name (str, optional): reference name, must be unique + """ + # Do not schedule if this event is already scheduled by the skill + if name not in self.scheduled_repeats: + # If only interval is given set to trigger in [interval] seconds + # from now. + if not when: + when = datetime.now() + timedelta(seconds=interval) + self._schedule_event(handler, when, data, name, interval) + else: + LOG.debug('The event is already scheduled, cancel previous ' + 'event if this scheduling should replace the last.') + + def update_scheduled_event(self, name, data=None): + """Change data of event. + + Arguments: + name (str): reference name of event (from original scheduling) + """ + data = data or {} + data = { + 'event': self._create_unique_name(name), + 'data': data + } + self.bus.emit(Message('mycroft.schedule.update_event', + data=data)) + + def cancel_scheduled_event(self, name): + """Cancel a pending event. The event will no longer be scheduled + to be executed + + Arguments: + name (str): reference name of event (from original scheduling) + """ + unique_name = self._create_unique_name(name) + data = {'event': unique_name} + if name in self.scheduled_repeats: + self.scheduled_repeats.remove(name) + if self.events.remove(unique_name): + self.bus.emit(Message('mycroft.scheduler.remove_event', + data=data)) + + def get_scheduled_event_status(self, name): + """Get scheduled event data and return the amount of time left + + Arguments: + name (str): reference name of event (from original scheduling) + + Returns: + int: the time left in seconds + + Raises: + Exception: Raised if event is not found + """ + event_name = self._create_unique_name(name) + data = {'name': event_name} + + reply_name = 'mycroft.event_status.callback.{}'.format(event_name) + msg = Message('mycroft.scheduler.get_event', data=data) + status = self.bus.wait_for_response(msg, reply_type=reply_name) + + if status: + event_time = int(status.data[0][0]) + current_time = int(time.time()) + time_left_in_seconds = event_time - current_time + LOG.info(time_left_in_seconds) + return time_left_in_seconds + else: + raise Exception("Event Status Messagebus Timeout") + + def cancel_all_repeating_events(self): + """Cancel any repeating events started by the skill.""" + # NOTE: Gotta make a copy of the list due to the removes that happen + # in cancel_scheduled_event(). + for e in list(self.scheduled_repeats): + self.cancel_scheduled_event(e) + + def shutdown(self): + """Shutdown the interface unregistering any event handlers.""" + self.cancel_all_repeating_events() + self.events.clear() diff --git a/mycroft/skills/intent_service.py b/mycroft/skills/intent_service.py index e5a2c600d92a..6696f406211f 100644 --- a/mycroft/skills/intent_service.py +++ b/mycroft/skills/intent_service.py @@ -19,12 +19,12 @@ from mycroft.configuration import Configuration from mycroft.messagebus.message import Message -from mycroft.skills.core import open_intent_envelope from mycroft.util.lang import set_active_lang from mycroft.util.log import LOG from mycroft.util.parse import normalize from mycroft.metrics import report_timing, Stopwatch from mycroft.skills.padatious_service import PadatiousService +from .intent_service_interface import open_intent_envelope class AdaptIntent(IntentBuilder): diff --git a/mycroft/skills/intent_service_interface.py b/mycroft/skills/intent_service_interface.py new file mode 100644 index 000000000000..b4050f6ff522 --- /dev/null +++ b/mycroft/skills/intent_service_interface.py @@ -0,0 +1,168 @@ +# Copyright 2018 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""The intent service interface offers a unified wrapper class for the +Intent Service. Including both adapt and padatious. +""" +from os.path import exists + +from adapt.intent import Intent + +from mycroft.messagebus.message import Message + + +class IntentServiceInterface: + """Interface to communicate with the Mycroft intent service. + + This class wraps the messagebus interface of the intent service allowing + for easier interaction with the service. It wraps both the Adapt and + Precise parts of the intent services. + """ + def __init__(self, bus=None): + self.bus = bus + self.registered_intents = [] + + def set_bus(self, bus): + self.bus = bus + + def register_adapt_keyword(self, vocab_type, entity, aliases=None): + """Send a message to the intent service to add an Adapt keyword. + + vocab_type(str): Keyword reference + entity (str): Primary keyword + aliases (list): List of alternative kewords + """ + aliases = aliases or [] + self.bus.emit(Message("register_vocab", + {'start': entity, 'end': vocab_type})) + for alias in aliases: + self.bus.emit(Message("register_vocab", { + 'start': alias, 'end': vocab_type, 'alias_of': entity + })) + + def register_adapt_regex(self, regex): + """Register a regex with the intent service. + + Arguments: + regex (str): Regex to be registered, (Adapt extracts keyword + reference from named match group. + """ + self.bus.emit(Message("register_vocab", {'regex': regex})) + + def register_adapt_intent(self, name, intent_parser): + """Register an Adapt intent parser object. + + Serializes the intent_parser and sends it over the messagebus to + registered. + """ + self.bus.emit(Message("register_intent", intent_parser.__dict__)) + self.registered_intents.append((name, intent_parser)) + + def detach_intent(self, intent_name): + """Remove an intent from the intent service. + + Arguments: + intent_name(str): Intent reference + """ + self.bus.emit(Message("detach_intent", {"intent_name": intent_name})) + + def set_adapt_context(self, context, word, origin): + """Set an Adapt context. + + Arguments: + context (str): context keyword name + word (str): word to register + origin (str): original origin of the context (for cross context) + """ + self.bus.emit(Message('add_context', + {'context': context, 'word': word, + 'origin': origin})) + + def remove_adapt_context(self, context): + """Remove an active Adapt context. + + Arguments: + context(str): name of context to remove + """ + self.bus.emit(Message('remove_context', {'context': context})) + + def register_padatious_intent(self, intent_name, filename): + """Register a padatious intent file with Padatious. + + Arguments: + intent_name(str): intent identifier + filename(str): complete file path for entity file + """ + if not isinstance(filename, str): + raise ValueError('Filename path must be a string') + if not exists(filename): + raise FileNotFoundError('Unable to find "{}"'.format(filename)) + + data = {"file_name": filename, + "name": intent_name} + self.bus.emit(Message("padatious:register_intent", data)) + self.registered_intents.append((intent_name, data)) + + def register_padatious_entity(self, entity_name, filename): + """Register a padatious entity file with Padatious. + + Arguments: + entity_name(str): entity name + filename(str): complete file path for entity file + """ + if not isinstance(filename, str): + raise ValueError('Filename path must be a string') + if not exists(filename): + raise FileNotFoundError('Unable to find "{}"'.format(filename)) + + self.bus.emit(Message('padatious:register_entity', { + 'file_name': filename, + 'name': entity_name + })) + + def __iter__(self): + """Iterator over the registered intents. + + Returns an iterator returning name-handler pairs of the registered + intent handlers. + """ + return iter(self.registered_intents) + + def __contains__(self, val): + """Checks if an intent name has been registered.""" + return val in [i[0] for i in self.registered_intents] + + def get_intent(self, intent_name): + """Get intent from intent_name. + + Arguments: + intent_name (str): name to find. + + Returns: + Found intent or None if none were found. + """ + for name, intent in self: + if name == intent_name: + return intent + else: + return None + + +def open_intent_envelope(message): + """Convert dictionary received over messagebus to Intent.""" + intent_dict = message.data + return Intent(intent_dict.get('name'), + intent_dict.get('requires'), + intent_dict.get('at_least_one'), + intent_dict.get('optional')) diff --git a/mycroft/skills/mycroft_skill/__init__.py b/mycroft/skills/mycroft_skill/__init__.py new file mode 100644 index 000000000000..29e5b57d5802 --- /dev/null +++ b/mycroft/skills/mycroft_skill/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2019 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .mycroft_skill import MycroftSkill +from .event_container import get_handler_name +from .decorators import (intent_handler, intent_file_handler, + resting_screen_handler) diff --git a/mycroft/skills/mycroft_skill/decorators.py b/mycroft/skills/mycroft_skill/decorators.py new file mode 100644 index 000000000000..676e4478afb6 --- /dev/null +++ b/mycroft/skills/mycroft_skill/decorators.py @@ -0,0 +1,62 @@ +# Copyright 2019 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Decorators for use with MycroftSkill methods""" + + +def intent_handler(intent_parser): + """Decorator for adding a method as an intent handler.""" + + def real_decorator(func): + # Store the intent_parser inside the function + # This will be used later to call register_intent + if not hasattr(func, 'intents'): + func.intents = [] + func.intents.append(intent_parser) + return func + + return real_decorator + + +def intent_file_handler(intent_file): + """Decorator for adding a method as an intent file handler. + + This decorator is deprecated, use intent_handler for the same effect. + """ + + def real_decorator(func): + # Store the intent_file inside the function + # This will be used later to call register_intent_file + if not hasattr(func, 'intent_files'): + func.intent_files = [] + func.intent_files.append(intent_file) + return func + + return real_decorator + + +def resting_screen_handler(name): + """Decorator for adding a method as an resting screen handler. + + If selected will be shown on screen when device enters idle mode. + """ + + def real_decorator(func): + # Store the resting information inside the function + # This will be used later in register_resting_screen + if not hasattr(func, 'resting_handler'): + func.resting_handler = name + return func + + return real_decorator diff --git a/mycroft/skills/mycroft_skill/event_container.py b/mycroft/skills/mycroft_skill/event_container.py new file mode 100644 index 000000000000..220f62def4fa --- /dev/null +++ b/mycroft/skills/mycroft_skill/event_container.py @@ -0,0 +1,158 @@ +from inspect import signature + +from mycroft.messagebus.message import Message +from mycroft.metrics import Stopwatch, report_timing +from mycroft.util.log import LOG + +from ..skill_data import to_alnum + + +def unmunge_message(message, skill_id): + """Restore message keywords by removing the Letterified skill ID. + Arguments: + message (Message): Intent result message + skill_id (str): skill identifier + Returns: + Message without clear keywords + """ + if isinstance(message, Message) and isinstance(message.data, dict): + skill_id = to_alnum(skill_id) + for key in list(message.data.keys()): + if key.startswith(skill_id): + # replace the munged key with the real one + new_key = key[len(skill_id):] + message.data[new_key] = message.data.pop(key) + + return message + + +def get_handler_name(handler): + """Name (including class if available) of handler function. + + Arguments: + handler (function): Function to be named + + Returns: + string: handler name as string + """ + if '__self__' in dir(handler) and 'name' in dir(handler.__self__): + return handler.__self__.name + '.' + handler.__name__ + else: + return handler.__name__ + + +def create_wrapper(handler, skill_id, on_start, on_end, on_error): + """Create the default skill handler wrapper. + + This wrapper handles things like metrics, reporting handler start/stop + and errors. + handler (callable): method/function to call + skill_id: skill_id for associated skill + on_start (function): function to call before executing the handler + on_end (function): function to call after executing the handler + on_error (function): function to call for error reporting + """ + def wrapper(message): + stopwatch = Stopwatch() + try: + message = unmunge_message(message, skill_id) + if on_start: + on_start(message) + + with stopwatch: + if len(signature(handler).parameters) == 0: + handler() + else: + handler(message) + + except Exception as e: + if on_error: + on_error(e) + finally: + if on_end: + on_end(message) + + # Send timing metrics + context = message.context + if context and 'ident' in context: + report_timing(context['ident'], 'skill_handler', stopwatch, + {'handler': handler.__name__, + 'skill_id': skill_id}) + return wrapper + + +class EventContainer: + """Container tracking messagbus handlers. + + This container tracks events added by a skill, allowing unregestering + all events on shutdown. + """ + def __init__(self, bus=None): + self.bus = bus + self.events = [] + + def set_bus(self, bus): + self.bus = bus + + def add(self, name, handler, once=False): + """Create event handler for executing intent or other event. + + Arguments: + name (string): IntentParser name + handler (func): Method to call + once (bool, optional): Event handler will be removed after it has + been run once. + """ + def once_wrapper(message): + # Remove registered one-time handler before invoking, + # allowing them to re-schedule themselves. + handler(message) + self.remove(name) + + if handler: + if once: + self.bus.once(name, once_wrapper) + else: + self.bus.on(name, handler) + self.events.append((name, handler)) + + def remove(self, name): + """Removes an event from bus emitter and events list. + + Args: + name (string): Name of Intent or Scheduler Event + Returns: + bool: True if found and removed, False if not found + """ + print("Removing event {}".format(name)) + removed = False + for _name, _handler in list(self.events): + if name == _name: + try: + self.events.remove((_name, _handler)) + except ValueError: + LOG.error('Failed to remove event {}'.format(name)) + pass + removed = True + + # Because of function wrappers, the emitter doesn't always directly + # hold the _handler function, it sometimes holds something like + # 'wrapper(_handler)'. So a call like: + # self.bus.remove(_name, _handler) + # will not find it, leaving an event handler with that name left behind + # waiting to fire if it is ever re-installed and triggered. + # Remove all handlers with the given name, regardless of handler. + if removed: + self.bus.remove_all_listeners(name) + return removed + + def __iter__(self): + return iter(self.events) + + def clear(self): + """Unregister all registered handlers and clear the list of registered + events. + """ + for e, f in self.events: + self.bus.remove_all_listeners(e) + self.events = [] # Remove reference to wrappers diff --git a/mycroft/skills/mycroft_skill.py b/mycroft/skills/mycroft_skill/mycroft_skill.py similarity index 69% rename from mycroft/skills/mycroft_skill.py rename to mycroft/skills/mycroft_skill/mycroft_skill.py index 9247aa63d960..9f7e1c72fe93 100644 --- a/mycroft/skills/mycroft_skill.py +++ b/mycroft/skills/mycroft_skill/mycroft_skill.py @@ -18,11 +18,6 @@ import re import traceback import inspect -from inspect import ismethod, signature -import collections -import time -from datetime import datetime, timedelta -import csv from itertools import chain from adapt.intent import Intent, IntentBuilder from os import walk @@ -38,14 +33,18 @@ from mycroft.dialog import DialogLoader from mycroft.filesystem import FileSystemAccess from mycroft.messagebus.message import Message -from mycroft.metrics import report_metric, report_timing, Stopwatch -from mycroft.util import (camel_case_split, - resolve_resource_file, - play_audio_file) +from mycroft.metrics import report_metric +from mycroft.util import (resolve_resource_file, play_audio_file, + camel_case_split) from mycroft.util.log import LOG -from .settings import SkillSettings -from .skill_data import (load_vocabulary, load_regex, to_alnum, - munge_regex, munge_intent_parser, read_vocab_file) + +from ..settings import SkillSettings +from ..skill_data import (load_vocabulary, load_regex, to_alnum, + munge_regex, munge_intent_parser, read_vocab_file, + read_value_file, read_translated_file) +from ..event_scheduler import EventSchedulerInterface +from ..intent_service_interface import IntentServiceInterface +from .event_container import EventContainer, create_wrapper, get_handler_name def simple_trace(stack_trace): @@ -100,94 +99,6 @@ def dig_for_message(): return l['message'] -def unmunge_message(message, skill_id): - """Restore message keywords by removing the Letterified skill ID. - - Arguments: - message (Message): Intent result message - skill_id (str): skill identifier - - Returns: - Message without clear keywords - """ - if isinstance(message, Message) and isinstance(message.data, dict): - skill_id = to_alnum(skill_id) - for key in list(message.data.keys()): - if key.startswith(skill_id): - # replace the munged key with the real one - new_key = key[len(skill_id):] - message.data[new_key] = message.data.pop(key) - - return message - - -def open_intent_envelope(message): - """Convert dictionary received over messagebus to Intent.""" - intent_dict = message.data - return Intent(intent_dict.get('name'), - intent_dict.get('requires'), - intent_dict.get('at_least_one'), - intent_dict.get('optional')) - - -def get_handler_name(handler): - """Name (including class if available) of handler function. - - Arguments: - handler (function): Function to be named - - Returns: - string: handler name as string - """ - if '__self__' in dir(handler) and 'name' in dir(handler.__self__): - return handler.__self__.name + '.' + handler.__name__ - else: - return handler.__name__ - - -def intent_handler(intent_parser): - """Decorator for adding a method as an intent handler.""" - - def real_decorator(func): - # Store the intent_parser inside the function - # This will be used later to call register_intent - if not hasattr(func, 'intents'): - func.intents = [] - func.intents.append(intent_parser) - return func - - return real_decorator - - -def intent_file_handler(intent_file): - """Decorator for adding a method as an intent file handler.""" - - def real_decorator(func): - # Store the intent_file inside the function - # This will be used later to call register_intent_file - if not hasattr(func, 'intent_files'): - func.intent_files = [] - func.intent_files.append(intent_file) - return func - - return real_decorator - - -def resting_screen_handler(name): - """Decorator for adding a method as an resting screen handler. - - If selected will be shown on screen when device enters idle mode. - """ - def real_decorator(func): - # Store the resting information inside the function - # This will be used later in register_resting_screen - if not hasattr(func, 'resting_handler'): - func.resting_handler = name - return func - - return real_decorator - - class MycroftSkill: """Base class for mycroft skills providing common behaviour and parameters to all Skill implementations. @@ -203,10 +114,11 @@ class MycroftSkill: def __init__(self, name=None, bus=None, use_settings=True): self.name = name or self.__class__.__name__ self.resting_name = None + self.skill_id = '' # will be set from the path, so guaranteed unique # Get directory of skill - self._dir = dirname(abspath(sys.modules[self.__module__].__file__)) + self.root_dir = dirname(abspath(sys.modules[self.__module__].__file__)) if use_settings: - self.settings = SkillSettings(self._dir, self.name) + self.settings = SkillSettings(self.root_dir, self.name) else: self.settings = None @@ -220,19 +132,21 @@ def __init__(self, name=None, bus=None, use_settings=True): # TODO: 19.08 - Remove self._config = self.config_core.get(self.name) or {} self.dialog_renderer = None - self.root_dir = None #: skill root directory #: Filesystem access to skill specific folder. #: See mycroft.filesystem for details. self.file_system = FileSystemAccess(join('skills', self.name)) - self.registered_intents = [] + self.log = LOG.create_logger(self.name) #: Skill logger instance self.reload_skill = True #: allow reloading (default True) - self.events = [] - self.scheduled_repeats = [] - self.skill_id = '' # will be set from the path, so guaranteed unique + + self.events = EventContainer(bus) self.voc_match_cache = {} + # Delegator classes + self.event_scheduler = EventSchedulerInterface(self.name) + self.intent_service = IntentServiceInterface() + @property def enclosure(self): if self._enclosure: @@ -299,42 +213,39 @@ def bind(self, bus): """ if bus: self._bus = bus + self.events.set_bus(bus) + self.intent_service.set_bus(bus) + self.event_scheduler.set_bus(bus) + self.event_scheduler.set_id(self.skill_id) self._enclosure = EnclosureAPI(bus, self.name) - self.add_event('mycroft.stop', self.__handle_stop) - self.add_event('mycroft.skill.enable_intent', - self.handle_enable_intent) - self.add_event('mycroft.skill.disable_intent', - self.handle_disable_intent) - self.add_event("mycroft.skill.set_cross_context", - self.handle_set_cross_context) - self.add_event("mycroft.skill.remove_cross_context", - self.handle_remove_cross_context) - - # Trigger settings update if requested - self._add_light_event('mycroft.skills.settings.update', - self.settings.run_poll) - # Trigger Settings meta upload on pairing complete - self._add_light_event('mycroft.paired', self.settings.run_poll) - + self._register_system_event_handlers() # Intialize the SkillGui self.gui.setup_default_handlers() - def _add_light_event(self, msg_type, func): - """This adds an event handler that will automatically be unregistered - when the skill shutsdown but none of the metrics or error feedback will - be triggered. - - Arguments: - msg_tupe (str): Message type - func (Function): function to be invoked + def _register_system_event_handlers(self): + """Add all events allowing the standard interaction with the Mycroft + system. """ - self.bus.on(msg_type, func) - self.events.append((msg_type, func)) + self.add_event('mycroft.stop', self.__handle_stop) + self.add_event('mycroft.skill.enable_intent', + self.handle_enable_intent) + self.add_event('mycroft.skill.disable_intent', + self.handle_disable_intent) + self.add_event("mycroft.skill.set_cross_context", + self.handle_set_cross_context) + self.add_event("mycroft.skill.remove_cross_context", + self.handle_remove_cross_context) + + # Trigger settings update if requested + self.events.add('mycroft.skills.settings.update', + self.settings.run_poll) + # Trigger Settings meta upload on pairing complete + self.events.add('mycroft.paired', self.settings.run_poll) def detach(self): for (name, intent) in self.registered_intents: - name = str(self.skill_id) + ':' + name - self.bus.emit(Message("detach_intent", {"intent_name": name})) + name = '{}:{}'.format(self.skill_id, name) + self.intent_service.detach_intent(name) def initialize(self): """Perform any final setup needed for the skill. @@ -427,10 +338,7 @@ def on_fail(utterance): """ data = data or {} - def get_announcement(): - return self.dialog_renderer.render(dialog, data) - - if not get_announcement(): + if not self.dialog_renderer.render(dialog, data): raise ValueError('dialog message required') def on_fail_default(utterance): @@ -439,7 +347,7 @@ def on_fail_default(utterance): if on_fail: return self.dialog_renderer.render(on_fail, fail_data) else: - return get_announcement() + return self.dialog_renderer.render(dialog, data) def is_cancel(utterance): return self.voc_match(utterance, 'cancel') @@ -448,10 +356,25 @@ def validator_default(utterance): # accept anything except 'cancel' return not is_cancel(utterance) - validator = validator or validator_default on_fail_fn = on_fail if callable(on_fail) else on_fail_default + validator = validator or validator_default - self.speak(get_announcement(), expect_response=True, wait=True) + # Speak query and wait for user response + self.speak(self.dialog_renderer.render(dialog, data), + expect_response=True, wait=True) + return self._wait_response(is_cancel, validator, on_fail_fn, + num_retries) + + def _wait_response(self, is_cancel, validator, on_fail, num_retries): + """Loop until a valid response is received from the user or the retry + limit is reached. + + Arguments: + is_cancel (callable): function checking cancel criteria + validator (callbale): function checking for a valid response + on_fail (callable): function handling retries + + """ num_fails = 0 while True: response = self.__get_response() @@ -473,7 +396,7 @@ def validator_default(utterance): if 0 < num_retries < num_fails: return None - line = on_fail_fn(response) + line = on_fail(response) self.speak(line, expect_response=True) def ask_yesno(self, prompt, data=None): @@ -521,7 +444,7 @@ def voc_match(self, utt, voc_filename, lang=None): if cache_key not in self.voc_match_cache: # Check for both skill resources and mycroft-core resources voc = self.find_resource(voc_filename + '.voc', 'vocab') - if not voc: + if not voc: # Check for vocab in mycroft core resources voc = resolve_resource_file(join('text', lang, voc_filename + '.voc')) @@ -529,8 +452,8 @@ def voc_match(self, utt, voc_filename, lang=None): raise FileNotFoundError( 'Could not find {}.voc file'.format(voc_filename)) # load vocab and flatten into a simple list - vocab = list(chain(*read_vocab_file(voc))) - self.voc_match_cache[cache_key] = vocab + vocab = read_vocab_file(voc) + self.voc_match_cache[cache_key] = list(chain(*vocab)) if utt: # Check for matches against complete words return any([re.match(r'.*\b' + i + r'\b.*', utt) @@ -545,7 +468,7 @@ def report_metric(self, name, data): name (str): Name of metric. Must use only letters and hyphens data (dict): JSON dictionary to report. Must be valid JSON """ - report_metric(basename(self.root_dir) + ':' + name, data) + report_metric('{}:{}'.format(basename(self.root_dir), name), data) def send_email(self, title, body): """Send an email to the registered user's email. @@ -677,7 +600,7 @@ def find_resource(self, res_name, res_dirname=None): # Not found return None - def translate_namedvalues(self, name, delim=None): + def translate_namedvalues(self, name, delim=','): """Load translation dict containing names and values. This loads a simple CSV from the 'dialog' folders. @@ -692,26 +615,13 @@ def translate_namedvalues(self, name, delim=None): dict: name and value dictionary, or empty dict if load fails """ - delim = delim or ',' - result = collections.OrderedDict() if not name.endswith(".value"): name += ".value" try: filename = self.find_resource(name, 'dialog') - if filename: - with open(filename) as f: - reader = csv.reader(f, delimiter=delim) - for row in reader: - # skip blank or comment lines - if not row or row[0].startswith("#"): - continue - if len(row) != 2: - continue - - result[row[0]] = row[1] - - return result + return read_value_file(filename, delim) + except Exception: return {} @@ -755,12 +665,7 @@ def translate_list(self, list_name, data=None): def __translate_file(self, name, data): """Load and render lines from dialog//""" filename = self.find_resource(name, 'dialog') - if filename: - with open(filename) as f: - text = f.read().replace('{{', '{').replace('}}', '}') - return text.format(**data or {}).rstrip('\n').split('\n') - else: - return None + return read_translated_file(filename, data) def add_event(self, name, handler, handler_info=None, once=False): """Create event handler for executing intent or other event. @@ -773,58 +678,37 @@ def add_event(self, name, handler, handler_info=None, once=False): once (bool, optional): Event handler will be removed after it has been run once. """ - - def wrapper(message): - skill_data = {'name': get_handler_name(handler)} - stopwatch = Stopwatch() - try: - message = unmunge_message(message, self.skill_id) - # Indicate that the skill handler is starting - if handler_info: - # Indicate that the skill handler is starting if requested - msg_type = handler_info + '.start' - self.bus.emit(message.reply(msg_type, skill_data)) - - if once: - # Remove registered one-time handler before invoking, - # allowing them to re-schedule themselves. - self.remove_event(name) - - with stopwatch: - if len(signature(handler).parameters) == 0: - handler() - else: - handler(message) - self.settings.store() # Store settings if they've changed - - except Exception as e: - # Convert "MyFancySkill" to "My Fancy Skill" for speaking - handler_name = camel_case_split(self.name) - msg_data = {'skill': handler_name} - msg = dialog.get('skill.error', self.lang, msg_data) - self.speak(msg) - LOG.exception(msg) - # append exception information in message - skill_data['exception'] = repr(e) - finally: - # Indicate that the skill handler has completed - if handler_info: - msg_type = handler_info + '.complete' - self.bus.emit(message.reply(msg_type, skill_data)) - - # Send timing metrics - context = message.context - if context and 'ident' in context: - report_timing(context['ident'], 'skill_handler', stopwatch, - {'handler': handler.__name__, - 'skill_id': self.skill_id}) - - if handler: - if once: - self.bus.once(name, wrapper) - else: - self.bus.on(name, wrapper) - self.events.append((name, wrapper)) + skill_data = {'name': get_handler_name(handler)} + + def on_error(e): + """Speak and log the error.""" + # Convert "MyFancySkill" to "My Fancy Skill" for speaking + handler_name = camel_case_split(self.name) + msg_data = {'skill': handler_name} + msg = dialog.get('skill.error', self.lang, msg_data) + self.speak(msg) + LOG.exception(msg) + # append exception information in message + skill_data['exception'] = repr(e) + + def on_start(message): + """Indicate that the skill handler is starting.""" + if handler_info: + # Indicate that the skill handler is starting if requested + msg_type = handler_info + '.start' + self.bus.emit(message.reply(msg_type, skill_data)) + + def on_end(message): + """Store settings and indicate that the skill handler has completed + """ + self.settings.store() # Store settings if they've changed + if handler_info: + msg_type = handler_info + '.complete' + self.bus.emit(message.reply(msg_type, skill_data)) + + wrapper = create_wrapper(handler, self.skill_id, on_start, on_end, + on_error) + return self.events.add(name, wrapper, once) def remove_event(self, name): """Removes an event from bus emitter and events list. @@ -834,45 +718,40 @@ def remove_event(self, name): Returns: bool: True if found and removed, False if not found """ - removed = False - for _name, _handler in list(self.events): - if name == _name: - try: - self.events.remove((_name, _handler)) - except ValueError: - pass - removed = True - - # Because of function wrappers, the emitter doesn't always directly - # hold the _handler function, it sometimes holds something like - # 'wrapper(_handler)'. So a call like: - # self.bus.remove(_name, _handler) - # will not find it, leaving an event handler with that name left behind - # waiting to fire if it is ever re-installed and triggered. - # Remove all handlers with the given name, regardless of handler. - if removed: - self.bus.remove_all_listeners(name) - return removed + return self.events.remove(name) + + def _register_adapt_intent(self, intent_parser, handler): + """Register an adapt intent. + + Arguments: + intent_parser: Intent object to parse utterance for the handler. + handler (func): function to register with intent + """ + # Default to the handler's function name if none given + name = intent_parser.name or handler.__name__ + munge_intent_parser(intent_parser, name, self.skill_id) + self.intent_service.register_adapt_intent(name, intent_parser) + if handler: + self.add_event(intent_parser.name, handler, + 'mycroft.skill.handler') def register_intent(self, intent_parser, handler): """Register an Intent with the intent service. Arguments: - intent_parser: Intent or IntentBuilder object to parse - utterance for the handler. + intent_parser: Intent, IntentBuilder object or padatious intent + file to parse utterance for the handler. handler (func): function to register with intent """ if isinstance(intent_parser, IntentBuilder): intent_parser = intent_parser.build() + if (isinstance(intent_parser, str) and + intent_parser.endswith('.intent')): + return self.register_intent_file(intent_parser, handler) elif not isinstance(intent_parser, Intent): raise ValueError('"' + str(intent_parser) + '" is not an Intent') - # Default to the handler's function name if none given - name = intent_parser.name or handler.__name__ - munge_intent_parser(intent_parser, name, self.skill_id) - self.bus.emit(Message("register_intent", intent_parser.__dict__)) - self.registered_intents.append((name, intent_parser)) - self.add_event(intent_parser.name, handler, 'mycroft.skill.handler') + return self._register_adapt_intent(intent_parser, handler) def register_intent_file(self, intent_file, handler): """Register an Intent file with the intent service. @@ -901,20 +780,11 @@ def register_intent_file(self, intent_file, handler): '.intent' handler: function to register with intent """ - name = str(self.skill_id) + ':' + intent_file - + name = '{}:{}'.format(self.skill_id, intent_file) filename = self.find_resource(intent_file, 'vocab') if not filename: - raise FileNotFoundError( - 'Unable to find "' + str(intent_file) + '"' - ) - - data = { - "file_name": filename, - "name": name - } - self.bus.emit(Message("padatious:register_intent", data)) - self.registered_intents.append((intent_file, data)) + raise FileNotFoundError('Unable to find "{}"'.format(intent_file)) + self.intent_service.register_padatious_intent(name, filename) self.add_event(name, handler, 'mycroft.skill.handler') def register_entity_file(self, entity_file): @@ -936,18 +806,12 @@ def register_entity_file(self, entity_file): """ if entity_file.endswith('.entity'): entity_file = entity_file.replace('.entity', '') - filename = self.find_resource(entity_file + ".entity", 'vocab') if not filename: - raise FileNotFoundError( - 'Unable to find "' + entity_file + '.entity"' - ) - name = str(self.skill_id) + ':' + entity_file - - self.bus.emit(Message("padatious:register_entity", { - "file_name": filename, - "name": name - })) + raise FileNotFoundError('Unable to find "{}"'.format(entity_file)) + + name = '{}:{}'.format(self.skill_id, entity_file) + self.intent_service.register_padatious_entity(name, filename) def handle_enable_intent(self, message): """Listener to enable a registered intent if it belongs to this skill. @@ -974,16 +838,15 @@ def disable_intent(self, intent_name): Returns: bool: True if disabled, False if it wasn't registered """ - names = [intent_tuple[0] for intent_tuple in self.registered_intents] - if intent_name in names: + if intent_name in self.intent_service: LOG.debug('Disabling intent ' + intent_name) - name = str(self.skill_id) + ':' + intent_name - self.bus.emit(Message("detach_intent", {"intent_name": name})) + name = '{}:{}'.format(self.skill_id, intent_name) + self.intent_service.detach_intent(name) return True - - LOG.error('Could not disable ' + intent_name + - ', it hasn\'t been registered.') - return False + else: + LOG.error('Could not disable ' + '{}, it hasn\'t been registered.'.format(intent_name)) + return False def enable_intent(self, intent_name): """(Re)Enable a registered intent if it belongs to this skill. @@ -994,24 +857,21 @@ def enable_intent(self, intent_name): Returns: bool: True if enabled, False if it wasn't registered """ - names = [intent[0] for intent in self.registered_intents] - intents = [intent[1] for intent in self.registered_intents] - if intent_name in names: - intent = intents[names.index(intent_name)] - self.registered_intents.remove((intent_name, intent)) + intent = self.intent_service.get_intent(intent_name) + if intent: if ".intent" in intent_name: self.register_intent_file(intent_name, None) else: intent.name = intent_name self.register_intent(intent, None) - LOG.debug('Enabling intent ' + intent_name) + LOG.debug('Enabling intent {}'.format(intent_name)) return True + else: + LOG.error('Could not enable ' + '{}, it hasn\'t been registered.'.format(intent_name)) + return False - LOG.error('Could not enable ' + intent_name + ', it hasn\'t been ' - 'registered.') - return False - - def set_context(self, context, word='', origin=None): + def set_context(self, context, word='', origin=''): """Add context to intent service Arguments: @@ -1019,15 +879,12 @@ def set_context(self, context, word='', origin=None): word: word connected to keyword """ if not isinstance(context, str): - raise ValueError('context should be a string') + raise ValueError('Context should be a string') if not isinstance(word, str): - raise ValueError('word should be a string') + raise ValueError('Word should be a string') - origin = origin or '' context = to_alnum(self.skill_id) + context - self.bus.emit(Message('add_context', - {'context': context, 'word': word, - 'origin': origin})) + self.intent_service.set_adapt_context(context, word, origin) def handle_set_cross_context(self, message): """Add global context to intent service.""" @@ -1065,7 +922,7 @@ def remove_context(self, context): if not isinstance(context, str): raise ValueError('context should be a string') context = to_alnum(self.skill_id) + context - self.bus.emit(Message('remove_context', {'context': context})) + self.intent_service.remove_adapt_context(context) def register_vocabulary(self, entity, entity_type): """ Register a word to a keyword @@ -1085,7 +942,7 @@ def register_regex(self, regex_str): """ regex = munge_regex(regex_str, self.skill_id) re.compile(regex) # validate regex - self.bus.emit(Message('register_vocab', {'regex': regex})) + self.intent_service.register_adapt_regex(regex) def speak(self, utterance, expect_response=False, wait=False): """Speak a sentence. @@ -1103,10 +960,9 @@ def speak(self, utterance, expect_response=False, wait=False): data = {'utterance': utterance, 'expect_response': expect_response} message = dig_for_message() - if message: - self.bus.emit(message.reply("speak", data)) - else: - self.bus.emit(Message("speak", data)) + m = message.reply("speak", data) if message else Message("speak", data) + self.bus.emit(m) + if wait: wait_while_speaking() @@ -1127,6 +983,24 @@ def speak_dialog(self, key, data=None, expect_response=False, wait=False): self.speak(self.dialog_renderer.render(key, data), expect_response, wait) + def acknowledge(self): + """Acknowledge a successful request. + + This method plays a sound to acknowledge a request that does not + require a verbal response. This is intended to provide simple feedback + to the user that their request was handled successfully. + """ + audio_file = resolve_resource_file( + self.config_core.get('sounds').get('acknowledge')) + + if not audio_file: + LOG.warning("Could not find 'acknowledge' audio file!") + return + + process = play_audio_file(audio_file) + if not process: + LOG.warning("Unable to play 'acknowledge' audio file!") + def init_dialog(self, root_directory): # If "/dialog/" exists, load from there. Otherwise # load dialog from "/locale/" @@ -1139,13 +1013,13 @@ def init_dialog(self, root_directory): else: LOG.debug('No dialog loaded') - def load_data_files(self, root_directory): + def load_data_files(self, root_directory=None): """Load data files (intents, dialogs, etc). Arguments: root_directory (str): root folder to use when loading files. """ - self.root_dir = root_directory + root_directory = root_directory or self.root_dir self.init_dialog(root_directory) self.load_vocab_files(root_directory) self.load_regex_files(root_directory) @@ -1156,27 +1030,41 @@ def load_vocab_files(self, root_directory): Arguments: root_directory (str): root folder to use when loading files """ + keywords = [] vocab_dir = join(root_directory, 'vocab', self.lang) + locale_dir = join(root_directory, 'locale', self.lang) if exists(vocab_dir): - load_vocabulary(vocab_dir, self.bus, self.skill_id) - elif exists(join(root_directory, 'locale', self.lang)): - load_vocabulary(join(root_directory, 'locale', self.lang), - self.bus, self.skill_id) + keywords = load_vocabulary(vocab_dir, self.skill_id) + elif exists(locale_dir): + keywords = load_vocabulary(locale_dir, self.skill_id) else: LOG.debug('No vocab loaded') + # For each found intent register the default along with any aliases + for vocab_type in keywords: + for line in keywords[vocab_type]: + entity = line[0] + aliases = line[1:] + self.intent_service.register_adapt_keyword(vocab_type, + entity, + aliases) + def load_regex_files(self, root_directory): - """ Load regex files found under root_directory. + """ Load regex files found under the skill directory. Arguments: root_directory (str): root folder to use when loading files """ + regexes = [] regex_dir = join(root_directory, 'regex', self.lang) + locale_dir = join(root_directory, 'locale', self.lang) if exists(regex_dir): - load_regex(regex_dir, self.bus, self.skill_id) - elif exists(join(root_directory, 'locale', self.lang)): - load_regex(join(root_directory, 'locale', self.lang), - self.bus, self.skill_id) + regexes = load_regex(regex_dir, self.skill_id) + elif exists(locale_dir): + regexes = load_regex(locale_dir, self.skill_id) + + for regex in regexes: + self.intent_service.register_adapt_regex(regex) def __handle_stop(self, event): """Handler for the "mycroft.stop" signal. Runs the user defined @@ -1192,7 +1080,7 @@ def __stop_timeout(): try: if self.stop(): self.bus.emit(Message("mycroft.stop.handled", - {"by": "skill:"+str(self.skill_id)})) + {"by": "skill:" + self.skill_id})) timer.cancel() except Exception: timer.cancel() @@ -1223,7 +1111,7 @@ def default_shutdown(self): LOG.error('Skill specific shutdown function encountered ' 'an error: {}'.format(repr(e))) # Store settings - if exists(self._dir): + if exists(self.root_dir): self.settings.store() self.settings.stop_polling() @@ -1231,10 +1119,8 @@ def default_shutdown(self): self.gui.clear() # removing events - self.cancel_all_repeating_events() - for e, f in self.events: - self.bus.remove(e, f) - self.events = [] # Remove reference to wrappers + self.event_scheduler.shutdown() + self.events.clear() self.bus.emit( Message("detach_skill", {"skill_id": str(self.skill_id) + ":"})) @@ -1244,40 +1130,6 @@ def default_shutdown(self): LOG.error("Failed to stop skill: {}".format(self.name), exc_info=True) - def _unique_name(self, name): - """Return a name unique to this skill using the format - [skill_id]:[name]. - - Arguments: - name: Name to use internally - - Returns: - str: name unique to this skill - """ - return str(self.skill_id) + ':' + (name or '') - - def _schedule_event(self, handler, when, data=None, name=None, - repeat=None): - """Underlying method for schedule_event and schedule_repeating_event. - - Takes scheduling information and sends it off on the message bus. - """ - if not name: - name = self.name + handler.__name__ - unique_name = self._unique_name(name) - if repeat: - self.scheduled_repeats.append(name) # store "friendly name" - - data = data or {} - self.add_event(unique_name, handler, once=not repeat) - event_data = {} - event_data['time'] = time.mktime(when.timetuple()) - event_data['event'] = unique_name - event_data['repeat'] = repeat - event_data['data'] = data - self.bus.emit(Message('mycroft.scheduler.schedule_event', - data=event_data)) - def schedule_event(self, handler, when, data=None, name=None): """Schedule a single-shot event. @@ -1292,10 +1144,7 @@ def schedule_event(self, handler, when, data=None, name=None): previously scheduled event of the same name. """ - data = data or {} - if isinstance(when, (int, float)): - when = datetime.now() + timedelta(seconds=when) - self._schedule_event(handler, when, data, name) + return self.event_scheduler.schedule_event(handler, when, data, name) def schedule_repeating_event(self, handler, when, frequency, data=None, name=None): @@ -1311,15 +1160,9 @@ def schedule_repeating_event(self, handler, when, frequency, data (dict, optional): data to send when the handler is called name (str, optional): reference name, must be unique """ - # Do not schedule if this event is already scheduled by the skill - if name not in self.scheduled_repeats: - data = data or {} - if not when: - when = datetime.now() + timedelta(seconds=frequency) - self._schedule_event(handler, when, data, name, frequency) - else: - LOG.debug('The event is already scheduled, cancel previous ' - 'event if this scheduling should replace the last.') + return self.event_scheduler.schedule_repeating_event(handler, when, + frequency, data, + name) def update_scheduled_event(self, name, data=None): """Change data of event. @@ -1327,12 +1170,7 @@ def update_scheduled_event(self, name, data=None): Arguments: name (str): reference name of event (from original scheduling) """ - data = data or {} - data = { - 'event': self._unique_name(name), - 'data': data - } - self.bus.emit(Message('mycroft.schedule.update_event', data=data)) + return self.event_scheduler.update_scheduled_event(name, data) def cancel_scheduled_event(self, name): """Cancel a pending event. The event will no longer be scheduled @@ -1341,13 +1179,7 @@ def cancel_scheduled_event(self, name): Arguments: name (str): reference name of event (from original scheduling) """ - unique_name = self._unique_name(name) - data = {'event': unique_name} - if name in self.scheduled_repeats: - self.scheduled_repeats.remove(name) - if self.remove_event(unique_name): - self.bus.emit(Message('mycroft.scheduler.remove_event', - data=data)) + return self.event_scheduler.cancel_scheduled_event(name) def get_scheduled_event_status(self, name): """Get scheduled event data and return the amount of time left @@ -1361,55 +1193,8 @@ def get_scheduled_event_status(self, name): Raises: Exception: Raised if event is not found """ - event_name = self._unique_name(name) - data = {'name': event_name} - - # making event_status an object so it's refrence can be changed - event_status = None - finished_callback = False - - def callback(message): - nonlocal event_status - nonlocal finished_callback - if message.data is not None: - event_time = int(message.data[0][0]) - current_time = int(time.time()) - time_left_in_seconds = event_time - current_time - event_status = time_left_in_seconds - finished_callback = True - - emitter_name = 'mycroft.event_status.callback.{}'.format(event_name) - self.bus.once(emitter_name, callback) - self.bus.emit(Message('mycroft.scheduler.get_event', data=data)) - - start_wait = time.time() - while finished_callback is False and time.time() - start_wait < 3.0: - time.sleep(0.1) - if time.time() - start_wait > 3.0: - raise Exception("Event Status Messagebus Timeout") - return event_status + return self.event_scheduler.get_scheduled_event_status(name) def cancel_all_repeating_events(self): """Cancel any repeating events started by the skill.""" - # NOTE: Gotta make a copy of the list due to the removes that happen - # in cancel_scheduled_event(). - for e in list(self.scheduled_repeats): - self.cancel_scheduled_event(e) - - def acknowledge(self): - """Acknowledge a successful request. - - This method plays a sound to acknowledge a request that does not - require a verbal response. This is intended to provide simple feedback - to the user that their request was handled successfully. - """ - audio_file = resolve_resource_file( - self.config_core.get('sounds').get('acknowledge')) - - if not audio_file: - LOG.warning("Could not find 'acknowledge' audio file!") - return - - process = play_audio_file(audio_file) - if not process: - LOG.warning("Unable to play 'acknowledge' audio file!") + return self.event_scheduler.cancel_all_repeating_events() diff --git a/mycroft/skills/skill_data.py b/mycroft/skills/skill_data.py index f0730fca8396..9c7924590c67 100644 --- a/mycroft/skills/skill_data.py +++ b/mycroft/skills/skill_data.py @@ -13,13 +13,15 @@ # limitations under the License. # -"""Module containing methods needed to load skill -data such as dialogs, intents and regular expressions. +"""Module containing methods needed to load skill data such as intents and +regular expressions. """ from os import walk from os.path import splitext, join import re +import csv +import collections from mycroft.messagebus.message import Message from mycroft.util.format import expand_options @@ -47,64 +49,49 @@ def read_vocab_file(path): return vocab -def load_vocab_from_file(path, vocab_type, bus): - """Load Mycroft vocabulary from file - The vocab is sent to the intent handler using the message bus - - Args: - path: path to vocabulary file (*.voc) - vocab_type: keyword name - bus: Mycroft messagebus connection - skill_id(str): skill id - """ - if path.endswith('.voc'): - for parts in read_vocab_file(path): - entity = parts[0] - bus.emit(Message("register_vocab", { - 'start': entity, 'end': vocab_type - })) - for alias in parts[1:]: - bus.emit(Message("register_vocab", { - 'start': alias, 'end': vocab_type, 'alias_of': entity - })) - - -def load_regex_from_file(path, bus, skill_id): +def load_regex_from_file(path, skill_id): """Load regex from file The regex is sent to the intent handler using the message bus Args: path: path to vocabulary file (*.voc) - bus: Mycroft messagebus connection + skill_id: skill_id to the regex is tied to """ + regexes = [] if path.endswith('.rx'): with open(path, 'r', encoding='utf8') as reg_file: for line in reg_file.readlines(): if line.startswith("#"): continue - re.compile(munge_regex(line.strip(), skill_id)) - bus.emit( - Message("register_vocab", - {'regex': munge_regex(line.strip(), skill_id)})) + regex = munge_regex(line.strip(), skill_id) + # Raise error if regex can't be compiled + re.compile(regex) + regexes.append(regex) + + return regexes -def load_vocabulary(basedir, bus, skill_id): +def load_vocabulary(basedir, skill_id): """Load vocabulary from all files in the specified directory. - Args: + Arguments: basedir (str): path of directory to load from (will recurse) - bus (messagebus emitter): messagebus instance used to send the vocab to - the intent service skill_id: skill the data belongs to + Returns: + dict with intent_type as keys and list of list of lists as value. """ + vocabs = {} for path, _, files in walk(basedir): for f in files: if f.endswith(".voc"): vocab_type = to_alnum(skill_id) + splitext(f)[0] - load_vocab_from_file(join(path, f), vocab_type, bus) + vocs = read_vocab_file(join(path, f)) + if vocs: + vocabs[vocab_type] = vocs + return vocabs -def load_regex(basedir, bus, skill_id): +def load_regex(basedir, skill_id): """Load regex from all files in the specified directory. Args: @@ -113,10 +100,12 @@ def load_regex(basedir, bus, skill_id): the intent service skill_id (str): skill identifier """ + regexes = [] for path, _, files in walk(basedir): for f in files: if f.endswith(".rx"): - load_regex_from_file(join(path, f), bus, skill_id) + regexes += load_regex_from_file(join(path, f), skill_id) + return regexes def to_alnum(skill_id): @@ -193,3 +182,49 @@ def munge_intent_parser(intent_parser, name, skill_id): element = [skill_id + e.replace(skill_id, '') for e in i] at_least_one.append(tuple(element)) intent_parser.at_least_one = at_least_one + + +def read_value_file(filename, delim): + """Read value file. + + The value file is a simple csv structure with a key and value. + + Arguments: + filename (str): file to read + delim (str): csv delimiter + + Returns: + OrderedDict with results. + """ + result = collections.OrderedDict() + + if filename: + with open(filename) as f: + reader = csv.reader(f, delimiter=delim) + for row in reader: + # skip blank or comment lines + if not row or row[0].startswith("#"): + continue + if len(row) != 2: + continue + + result[row[0]] = row[1] + return result + + +def read_translated_file(filename, data): + """Read a file inserting data. + + Arguments: + filename (str): file to read + data (dict): dictionary with data to insert into file + + Returns: + list of lines. + """ + if filename: + with open(filename) as f: + text = f.read().replace('{{', '{').replace('}}', '}') + return text.format(**data or {}).rstrip('\n').split('\n') + else: + return None diff --git a/mycroft/skills/skill_loader.py b/mycroft/skills/skill_loader.py index 004fa07b04e2..bd31fc46534f 100644 --- a/mycroft/skills/skill_loader.py +++ b/mycroft/skills/skill_loader.py @@ -223,7 +223,7 @@ def _create_skill_instance(self, skill_module): self.instance.settings.load_skill_settings_from_file() self.instance.bind(self.bus) try: - self.instance.load_data_files(self.skill_directory) + self.instance.load_data_files() # Set up intent handlers # TODO: can this be a public method? self.instance._register_decorated() diff --git a/test/unittests/skills/test_event_container.py b/test/unittests/skills/test_event_container.py new file mode 100644 index 000000000000..20d63f0c8dce --- /dev/null +++ b/test/unittests/skills/test_event_container.py @@ -0,0 +1,68 @@ +import unittest +from unittest import mock + +from mycroft.skills.mycroft_skill.event_container import EventContainer + + +def example_handler(message): + pass + + +class TestEventContainer(unittest.TestCase): + def test_init(self): + bus = mock.MagicMock() + + # Set bus via init + container = EventContainer(bus) + self.assertEqual(container.bus, bus) + + # Set bus using .set_bus + container = EventContainer(None) + self.assertEqual(container.bus, None) + container.set_bus(bus) + self.assertEqual(container.bus, bus) + + def test_add(self): + bus = mock.MagicMock() + container = EventContainer(bus) + self.assertEqual(len(container.events), 0) + + # Test add normal event handler + container.add('test1', example_handler) + self.assertTrue(bus.on.called) + + # Test add single shot event handler + container.add('test2', example_handler, once=True) + self.assertTrue(bus.once.called) + + # Verify correct content in event container + self.assertTrue(('test1', example_handler) in container.events) + self.assertTrue(('test2', example_handler) in container.events) + self.assertEqual(len(container.events), 2) + + def test_remove(self): + bus = mock.MagicMock() + container = EventContainer(bus) + self.assertEqual(len(container.events), 0) + + container.add('test1', example_handler) + container.add('test2', example_handler) + container.add('test3', example_handler) + self.assertEqual(len(container.events), 3) + + self.assertTrue(('test2', example_handler) in container.events) + container.remove('test2') + self.assertTrue(('test2', example_handler) not in container.events) + self.assertTrue(bus.remove_all_listeners.called) + + def test_clear(self): + bus = mock.MagicMock() + container = EventContainer(bus) + + container.add('test1', example_handler) + container.add('test2', example_handler) + container.add('test3', example_handler) + self.assertEqual(len(container.events), 3) + + container.clear() + self.assertEqual(len(container.events), 0) diff --git a/test/unittests/skills/test_intent_service_interface.py b/test/unittests/skills/test_intent_service_interface.py new file mode 100644 index 000000000000..c9bf35bdef62 --- /dev/null +++ b/test/unittests/skills/test_intent_service_interface.py @@ -0,0 +1,59 @@ +import unittest + +from mycroft.skills.intent_service_interface import IntentServiceInterface + + +class MockEmitter: + def __init__(self): + self.reset() + + def emit(self, message): + self.types.append(message.type) + self.results.append(message.data) + + def get_types(self): + return self.types + + def get_results(self): + return self.results + + def on(self, event, f): + pass + + def reset(self): + self.types = [] + self.results = [] + + +class FunctionTest(unittest.TestCase): + def check_emitter(self, result_list): + for type in self.emitter.get_types(): + self.assertEqual(type, 'register_vocab') + self.assertEqual(sorted(self.emitter.get_results(), + key=lambda d: sorted(d.items())), + sorted(result_list, key=lambda d: sorted(d.items()))) + self.emitter.reset() + + def setUp(self): + self.emitter = MockEmitter() + + def test_register_keyword(self): + intent_service = IntentServiceInterface(self.emitter) + intent_service.register_adapt_keyword('test_intent', 'test') + self.check_emitter([{'start': 'test', 'end': 'test_intent'}]) + + def test_register_keyword_with_aliases(self): + intent_service = IntentServiceInterface(self.emitter) + intent_service.register_adapt_keyword('test_intent', 'test', + ['test2', 'test3']) + self.check_emitter([{'start': 'test', 'end': 'test_intent'}, + {'start': 'test2', 'end': 'test_intent', + 'alias_of': 'test'}, + {'start': 'test3', 'end': 'test_intent', + 'alias_of': 'test'}, + ]) + + def test_register_regex(self): + intent_service = IntentServiceInterface(self.emitter) + intent_service.register_adapt_regex('.*') + self.check_emitter([{'regex': '.*'}]) diff --git a/test/unittests/skills/test_core.py b/test/unittests/skills/test_mycroft_skill.py similarity index 81% rename from test/unittests/skills/test_core.py rename to test/unittests/skills/test_mycroft_skill.py index c5420034ff5d..6fa509a3c3f7 100644 --- a/test/unittests/skills/test_core.py +++ b/test/unittests/skills/test_mycroft_skill.py @@ -21,20 +21,14 @@ from os.path import join, dirname, abspath from re import error from datetime import datetime +import json from mycroft.configuration import Configuration from mycroft.messagebus.message import Message -from mycroft.skills.skill_data import ( - load_regex_from_file, - load_regex, - load_vocab_from_file, - load_vocabulary -) -from mycroft.skills.core import ( - MycroftSkill, - open_intent_envelope, - resting_screen_handler -) +from mycroft.skills.skill_data import (load_regex_from_file, load_regex, + load_vocabulary, read_vocab_file) +from mycroft.skills.core import MycroftSkill, resting_screen_handler +from mycroft.skills.intent_service import open_intent_envelope from test.util import base_config @@ -63,6 +57,10 @@ def reset(self): self.results = [] +def vocab_base_path(): + return join(dirname(__file__), '..', 'vocab_test') + + class FunctionTest(unittest.TestCase): def test_resting_screen_handler(self): class T(MycroftSkill): @@ -86,28 +84,28 @@ class MycroftSkillTest(unittest.TestCase): def setUp(self): self.emitter.reset() - def check_vocab_from_file(self, filename, vocab_type=None, - result_list=None): - result_list = result_list or [] - load_vocab_from_file(join(self.vocab_path, filename), vocab_type, - self.emitter) - self.check_emitter(result_list) + def check_vocab(self, filename, results=None): + results = results or {} + intents = load_vocabulary(join(self.vocab_path, filename), 'A') + self.compare_dicts(intents, results) def check_regex_from_file(self, filename, result_list=None): result_list = result_list or [] regex_file = join(self.regex_path, filename) - load_regex_from_file(regex_file, self.emitter, 'A') - self.check_emitter(result_list) + self.assertEqual(sorted(load_regex_from_file(regex_file, 'A')), + sorted(result_list)) - def check_vocab(self, path, result_list=None): - result_list = result_list or [] - load_vocabulary(path, self.emitter, 'A') - self.check_emitter(result_list) + def compare_dicts(self, d1, d2): + self.assertEqual(json.dumps(d1, sort_keys=True), + json.dumps(d2, sort_keys=True)) + + def check_read_vocab_file(self, path, result_list=None): + resultlist = result_list or [] + self.assertEqual(sorted(read_vocab_file(path)), sorted(result_list)) def check_regex(self, path, result_list=None): result_list = result_list or [] - load_regex(path, self.emitter, 'A') - self.check_emitter(result_list) + self.assertEqual(sorted(load_regex(path, 'A')), sorted(result_list)) def check_emitter(self, result_list): for type in self.emitter.get_types(): @@ -119,12 +117,12 @@ def check_emitter(self, result_list): def test_load_regex_from_file_single(self): self.check_regex_from_file('valid/single.rx', - [{'regex': '(?P.*)'}]) + ['(?P.*)']) def test_load_regex_from_file_multiple(self): self.check_regex_from_file('valid/multiple.rx', - [{'regex': '(?P.*)'}, - {'regex': '(?P.*)'}]) + ['(?P.*)', + '(?P.*)']) def test_load_regex_from_file_none(self): self.check_regex_from_file('invalid/none.rx') @@ -139,68 +137,48 @@ def test_load_regex_from_file_does_not_exist(self): def test_load_regex_full(self): self.check_regex(join(self.regex_path, 'valid'), - [{'regex': '(?P.*)'}, - {'regex': '(?P.*)'}, - {'regex': '(?P.*)'}]) + ['(?P.*)', + '(?P.*)', + '(?P.*)']) def test_load_regex_empty(self): - self.check_regex(join(dirname(__file__), - 'empty_dir')) + self.check_regex(join(dirname(__file__), 'empty_dir')) def test_load_regex_fail(self): try: - self.check_regex(join(dirname(__file__), - 'regex_test_fail')) + self.check_regex(join(dirname(__file__), 'regex_test_fail')) except OSError as e: self.assertEqual(e.strerror, 'No such file or directory') - def test_load_vocab_from_file_single(self): - self.check_vocab_from_file('valid/single.voc', 'test_type', - [{'start': 'test', 'end': 'test_type'}]) + def test_load_vocab_file_single(self): + self.check_read_vocab_file(join(vocab_base_path(), 'valid/single.voc'), + [['test']]) def test_load_vocab_from_file_single_alias(self): - self.check_vocab_from_file('valid/singlealias.voc', 'test_type', - [{'start': 'water', 'end': 'test_type'}, - {'start': 'watering', 'end': 'test_type', - 'alias_of': 'water'}]) - - def test_load_vocab_from_file_multiple(self): - self.check_vocab_from_file('valid/multiple.voc', 'test_type', - [{'start': 'animal', 'end': 'test_type'}, - {'start': 'animals', 'end': 'test_type'}]) + self.check_read_vocab_file(join(vocab_base_path(), + 'valid/singlealias.voc'), + [['water', 'watering']]) def test_load_vocab_from_file_multiple_alias(self): - self.check_vocab_from_file('valid/multiplealias.voc', 'test_type', - [{'start': 'chair', 'end': 'test_type'}, - {'start': 'chairs', 'end': 'test_type', - 'alias_of': 'chair'}, - {'start': 'table', 'end': 'test_type'}, - {'start': 'tables', 'end': 'test_type', - 'alias_of': 'table'}]) - - def test_load_vocab_from_file_none(self): - self.check_vocab_from_file('none.voc') + self.check_read_vocab_file(join(vocab_base_path(), + 'valid/multiplealias.voc'), + [['chair', 'chairs'], ['table', 'tables']]) def test_load_vocab_from_file_does_not_exist(self): try: - self.check_vocab_from_file('does_not_exist.voc') + self.check_read_vocab_file('does_not_exist.voc') except IOError as e: self.assertEqual(e.strerror, 'No such file or directory') def test_load_vocab_full(self): self.check_vocab(join(self.vocab_path, 'valid'), - [{'start': 'test', 'end': 'Asingle'}, - {'start': 'water', 'end': 'Asinglealias'}, - {'start': 'watering', 'end': 'Asinglealias', - 'alias_of': 'water'}, - {'start': 'animal', 'end': 'Amultiple'}, - {'start': 'animals', 'end': 'Amultiple'}, - {'start': 'chair', 'end': 'Amultiplealias'}, - {'start': 'chairs', 'end': 'Amultiplealias', - 'alias_of': 'chair'}, - {'start': 'table', 'end': 'Amultiplealias'}, - {'start': 'tables', 'end': 'Amultiplealias', - 'alias_of': 'table'}]) + { + 'Asingle': [['test']], + 'Asinglealias': [['water', 'watering']], + 'Amultiple': [['animal'], ['animals']], + 'Amultiplealias': [['chair', 'chairs'], + ['table', 'tables']] + }) def test_load_vocab_empty(self): self.check_vocab(join(dirname(__file__), 'empty_dir')) @@ -312,7 +290,13 @@ def check_register_object_file(self, types_list, result_list): self.emitter.reset() def test_register_intent_file(self): - s = SimpleSkill4() + self._test_intent_file(SimpleSkill4()) + + def test_register_intent_intent_file(self): + """Test register intent files using register_intent.""" + self._test_intent_file(SimpleSkill6()) + + def _test_intent_file(self, s): s.root_dir = abspath(join(dirname(__file__), 'intent_file')) s.bind(self.emitter) s.initialize() @@ -334,7 +318,6 @@ def test_register_intent_file(self): 'name': str(s.skill_id) + ':test_ent' } ] - self.check_register_object_file(expected_types, expected_results) def check_register_decorators(self, result_list): @@ -469,17 +452,20 @@ def test_add_scheduled_event(self): s.schedule_event(s.handler, datetime.now(), name='datetime_handler') # Check that the handler was registered with the emitter self.assertEqual(emitter.once.call_args[0][0], 'A:datetime_handler') - self.assertTrue('A:datetime_handler' in [e[0] for e in s.events]) + sched_events = [e[0] for e in s.event_scheduler.events] + self.assertTrue('A:datetime_handler' in sched_events) s.schedule_event(s.handler, 1, name='int_handler') # Check that the handler was registered with the emitter self.assertEqual(emitter.once.call_args[0][0], 'A:int_handler') - self.assertTrue('A:int_handler' in [e[0] for e in s.events]) + sched_events = [e[0] for e in s.event_scheduler.events] + self.assertTrue('A:int_handler' in sched_events) s.schedule_event(s.handler, .5, name='float_handler') # Check that the handler was registered with the emitter self.assertEqual(emitter.once.call_args[0][0], 'A:float_handler') - self.assertTrue('A:float_handler' in [e[0] for e in s.events]) + sched_events = [e[0] for e in s.event_scheduler.events] + self.assertTrue('A:float_handler' in sched_events) @patch.dict(Configuration._Configuration__config, BASE_CONF) def test_remove_scheduled_event(self): @@ -488,12 +474,15 @@ def test_remove_scheduled_event(self): s.bind(emitter) s.schedule_event(s.handler, datetime.now(), name='sched_handler1') # Check that the handler was registered with the emitter - self.assertTrue('A:sched_handler1' in [e[0] for e in s.events]) + events = [e[0] for e in s.event_scheduler.events] + print(events) + self.assertTrue('A:sched_handler1' in events) s.cancel_scheduled_event('sched_handler1') # Check that the handler was removed self.assertEqual(emitter.remove_all_listeners.call_args[0][0], 'A:sched_handler1') - self.assertTrue('A:sched_handler1' not in [e[0] for e in s.events]) + events = [e[0] for e in s.event_scheduler.events] + self.assertTrue('A:sched_handler1' not in events) @patch.dict(Configuration._Configuration__config, BASE_CONF) def test_run_scheduled_event(self): @@ -599,3 +588,15 @@ def handler(self, message): def stop(self): pass + + +class SimpleSkill6(_TestSkill): + """ Test skill for padatious intent """ + skill_id = 'A' + + def initialize(self): + self.register_intent('test.intent', self.handler) + self.register_entity_file('test_ent.entity') + + def handler(self, message): + pass