From 635c3f9ec72b9a404c15692fb54844015ef3528e Mon Sep 17 00:00:00 2001 From: NeonJarbas <59943014+NeonJarbas@users.noreply.github.com> Date: Sun, 7 Aug 2022 05:25:58 +0100 Subject: [PATCH] feat/integration_tests (#184) * feat/integration_tests authored-by: jarbasai --- .github/workflows/unit_tests.yml | 6 + .../ovos_tskill_abort/__init__.py | 60 ++++ .../locale/en-us/question.dialog | 1 + .../locale/en-us/test.intent | 1 + .../locale/en-us/test2.intent | 1 + .../locale/en-us/test3.intent | 1 + .../ovos_tskill_abort/readme.md | 1 + .../ovos_tskill_abort/setup.py | 23 ++ test/integrationtests/test_workshop.py | 273 ++++++++++++++++++ 9 files changed, 367 insertions(+) create mode 100644 test/integrationtests/ovos_tskill_abort/__init__.py create mode 100644 test/integrationtests/ovos_tskill_abort/locale/en-us/question.dialog create mode 100644 test/integrationtests/ovos_tskill_abort/locale/en-us/test.intent create mode 100644 test/integrationtests/ovos_tskill_abort/locale/en-us/test2.intent create mode 100644 test/integrationtests/ovos_tskill_abort/locale/en-us/test3.intent create mode 100644 test/integrationtests/ovos_tskill_abort/readme.md create mode 100755 test/integrationtests/ovos_tskill_abort/setup.py create mode 100644 test/integrationtests/test_workshop.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 592615a93e9d..f87a44609748 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -59,6 +59,12 @@ jobs: # NOTE: additional pytest invocations should also add the --cov-append flag # or they will overwrite previous invocations' coverage reports # (for an example, see OVOS Skill Manager's workflow) + - name: Run integration tests + run: | + pytest --cov-append --cov=mycroft --cov-report xml test/integrationtests + # NOTE: additional pytest invocations should also add the --cov-append flag + # or they will overwrite previous invocations' coverage reports + # (for an example, see OVOS Skill Manager's workflow) - name: Upload coverage env: CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}} diff --git a/test/integrationtests/ovos_tskill_abort/__init__.py b/test/integrationtests/ovos_tskill_abort/__init__.py new file mode 100644 index 000000000000..ba8d0456a2ea --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/__init__.py @@ -0,0 +1,60 @@ +from ovos_workshop.decorators import killable_intent +from ovos_workshop.skills.ovos import OVOSSkill +from mycroft.skills import intent_file_handler +from time import sleep + + +class TestAbortSkill(OVOSSkill): + """ + send "mycroft.skills.abort_question" and confirm only get_response is aborted + send "mycroft.skills.abort_execution" and confirm the full intent is aborted, except intent3 + send "my.own.abort.msg" and confirm intent3 is aborted + say "stop" and confirm all intents are aborted + """ + def __init__(self): + super(TestAbortSkill, self).__init__("KillableSkill") + self.my_special_var = "default" + self.stop_called = False + + def handle_intent_aborted(self): + self.speak("I am dead") + # handle any cleanup the skill might need, since intent was killed + # at an arbitrary place of code execution some variables etc. might + # end up in unexpected states + self.my_special_var = "default" + + @killable_intent(callback=handle_intent_aborted) + @intent_file_handler("test.intent") + def handle_test_abort_intent(self, message): + self.stop_called = False + self.my_special_var = "changed" + while True: + sleep(1) + self.speak("still here") + + @intent_file_handler("test2.intent") + @killable_intent(callback=handle_intent_aborted) + def handle_test_get_response_intent(self, message): + self.stop_called = False + self.my_special_var = "CHANGED" + ans = self.get_response("question", num_retries=99999) + self.log.debug("get_response returned: " + str(ans)) + if ans is None: + self.speak("question aborted") + + @killable_intent(msg="my.own.abort.msg", callback=handle_intent_aborted) + @intent_file_handler("test3.intent") + def handle_test_msg_intent(self, message): + self.stop_called = False + if self.my_special_var != "default": + self.speak("someone forgot to cleanup") + while True: + sleep(1) + self.speak("you can't abort me") + + def stop(self): + self.stop_called = True + + +def create_skill(): + return TestAbortSkill() diff --git a/test/integrationtests/ovos_tskill_abort/locale/en-us/question.dialog b/test/integrationtests/ovos_tskill_abort/locale/en-us/question.dialog new file mode 100644 index 000000000000..f0fb83cc4f33 --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/locale/en-us/question.dialog @@ -0,0 +1 @@ +this is a question \ No newline at end of file diff --git a/test/integrationtests/ovos_tskill_abort/locale/en-us/test.intent b/test/integrationtests/ovos_tskill_abort/locale/en-us/test.intent new file mode 100644 index 000000000000..30d74d258442 --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/locale/en-us/test.intent @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/test/integrationtests/ovos_tskill_abort/locale/en-us/test2.intent b/test/integrationtests/ovos_tskill_abort/locale/en-us/test2.intent new file mode 100644 index 000000000000..5161aff42996 --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/locale/en-us/test2.intent @@ -0,0 +1 @@ +test again \ No newline at end of file diff --git a/test/integrationtests/ovos_tskill_abort/locale/en-us/test3.intent b/test/integrationtests/ovos_tskill_abort/locale/en-us/test3.intent new file mode 100644 index 000000000000..1fec3fd265bf --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/locale/en-us/test3.intent @@ -0,0 +1 @@ +one more test \ No newline at end of file diff --git a/test/integrationtests/ovos_tskill_abort/readme.md b/test/integrationtests/ovos_tskill_abort/readme.md new file mode 100644 index 000000000000..add1af272c68 --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/readme.md @@ -0,0 +1 @@ +skill for testing https://github.com/OpenVoiceOS/ovos_utils/pull/34 \ No newline at end of file diff --git a/test/integrationtests/ovos_tskill_abort/setup.py b/test/integrationtests/ovos_tskill_abort/setup.py new file mode 100755 index 000000000000..fb2af9105142 --- /dev/null +++ b/test/integrationtests/ovos_tskill_abort/setup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from setuptools import setup + +# skill_id=package_name:SkillClass +PLUGIN_ENTRY_POINT = 'ovos-tskill-abort.openvoiceos=ovos_tskill_abort:TestAbortSkill' + +setup( + # this is the package name that goes on pip + name='ovos-tskill-abort', + version='0.0.1', + description='this is a OVOS test skill for the killable_intents decorator', + url='https://github.com/OpenVoiceOS/skill-abort-test', + author='JarbasAi', + author_email='jarbasai@mailfence.com', + license='Apache-2.0', + package_dir={"ovos_tskill_abort": ""}, + package_data={'ovos_tskill_abort': ['locale/*']}, + packages=['ovos_tskill_abort'], + include_package_data=True, + install_requires=["ovos-workshop"], + keywords='ovos skill plugin', + entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT} +) diff --git a/test/integrationtests/test_workshop.py b/test/integrationtests/test_workshop.py new file mode 100644 index 000000000000..604946f63447 --- /dev/null +++ b/test/integrationtests/test_workshop.py @@ -0,0 +1,273 @@ +import json +import unittest +from os.path import dirname +from time import sleep +from ovos_workshop.skills import OVOSSkill, MycroftSkill +from mycroft.skills.skill_loader import SkillLoader +from ovos_utils.messagebus import FakeBus, Message + +# tests taken from ovos_workshop + + +class TestSkill(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + self.bus.emitted_msgs = [] + + def get_msg(msg): + msg = json.loads(msg) + self.bus.emitted_msgs.append(msg) + + self.bus.on("message", get_msg) + + self.skill = SkillLoader(self.bus, f"{dirname(__file__)}/ovos_tskill_abort") + self.skill.skill_id = "abort.test" + self.bus.emitted_msgs = [] + + self.skill.load() + + def test_skill_id(self): + self.assertTrue(isinstance(self.skill.instance, OVOSSkill)) + self.assertTrue(isinstance(self.skill.instance, MycroftSkill)) + + self.assertEqual(self.skill.skill_id, "abort.test") + # if running in ovos-core every message will have the skill_id in context + for msg in self.bus.emitted_msgs: + if msg["type"] == 'mycroft.skills.loaded': # emitted by SkillLoader, not by skill + continue + self.assertEqual(msg["context"]["skill_id"], "abort.test") + + def test_intent_register(self): + padatious_intents = ["abort.test:test.intent", + "abort.test:test2.intent", + "abort.test:test3.intent"] + for msg in self.bus.emitted_msgs: + if msg["type"] == "padatious:register_intent": + self.assertTrue(msg["data"]["name"] in padatious_intents) + + def test_registered_events(self): + registered_events = [e[0] for e in self.skill.instance.events] + + # intent events + intent_triggers = [f"{self.skill.skill_id}:test.intent", + f"{self.skill.skill_id}:test2.intent", + f"{self.skill.skill_id}:test3.intent" + ] + for event in intent_triggers: + self.assertTrue(event in registered_events) + + # base skill class events shared with mycroft-core + default_skill = ["mycroft.skill.enable_intent", + "mycroft.skill.disable_intent", + "mycroft.skill.set_cross_context", + "mycroft.skill.remove_cross_context", + "mycroft.skills.settings.changed"] + for event in default_skill: + self.assertTrue(event in registered_events) + + # base skill class events exclusive to ovos-core + default_ovos = ["skill.converse.ping", + "skill.converse.request", + "intent.service.skills.activated", + "intent.service.skills.deactivated", + f"{self.skill.skill_id}.activate", + f"{self.skill.skill_id}.deactivate"] + for event in default_ovos: + self.assertTrue(event in registered_events) + + def tearDown(self) -> None: + self.skill.unload() + + +class TestKillableIntents(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + self.bus.emitted_msgs = [] + + def get_msg(msg): + m = json.loads(msg) + m.pop("context") + self.bus.emitted_msgs.append(m) + + self.bus.on("message", get_msg) + + self.skill = SkillLoader(self.bus, f"{dirname(__file__)}/ovos_tskill_abort") + self.skill.skill_id = "abort.test" + self.skill.load() + + def test_skills_abort_event(self): + self.bus.emitted_msgs = [] + # skill will enter a infinite loop unless aborted + self.assertTrue(self.skill.instance.my_special_var == "default") + self.bus.emit(Message(f"{self.skill.skill_id}:test.intent")) + sleep(2) + # check that intent triggered + start_msg = {'type': 'mycroft.skill.handler.start', + 'data': {'name': 'KillableSkill.handle_test_abort_intent'}} + speak_msg = {'type': 'speak', + 'data': {'utterance': 'still here', 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + self.assertIn(start_msg, self.bus.emitted_msgs) + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertTrue(self.skill.instance.my_special_var == "changed") + + # check that intent reacts to mycroft.skills.abort_execution + # eg, gui can emit this event if some option was selected + # on screen to abort the current voice interaction + self.bus.emitted_msgs = [] + self.bus.emit(Message(f"mycroft.skills.abort_execution")) + sleep(2) + + # check that stop method was called + self.assertTrue(self.skill.instance.stop_called) + + # check that TTS stop message was emmited + tts_stop = {'type': 'mycroft.audio.speech.stop', 'data': {}} + self.assertIn(tts_stop, self.bus.emitted_msgs) + + # check that cleanup callback was called + speak_msg = {'type': 'speak', + 'data': {'utterance': 'I am dead', 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertTrue(self.skill.instance.my_special_var == "default") + + # check that we are not getting speak messages anymore + self.bus.emitted_msgs = [] + sleep(2) + self.assertTrue(self.bus.emitted_msgs == []) + + def test_skill_stop(self): + self.bus.emitted_msgs = [] + # skill will enter a infinite loop unless aborted + self.assertTrue(self.skill.instance.my_special_var == "default") + self.bus.emit(Message(f"{self.skill.skill_id}:test.intent")) + sleep(2) + # check that intent triggered + start_msg = {'type': 'mycroft.skill.handler.start', + 'data': {'name': 'KillableSkill.handle_test_abort_intent'}} + speak_msg = {'type': 'speak', + 'data': {'utterance': 'still here', 'expect_response': False, + 'meta': {'skill': 'abort.test'}, 'lang': 'en-us'}} + self.assertIn(start_msg, self.bus.emitted_msgs) + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertTrue(self.skill.instance.my_special_var == "changed") + + # check that intent reacts to skill specific stop message + # this is also emitted on mycroft.stop if using OvosSkill class + self.bus.emitted_msgs = [] + self.bus.emit(Message(f"{self.skill.skill_id}.stop")) + sleep(2) + + # check that stop method was called + self.assertTrue(self.skill.instance.stop_called) + + # check that TTS stop message was emmited + tts_stop = {'type': 'mycroft.audio.speech.stop', 'data': {}} + self.assertIn(tts_stop, self.bus.emitted_msgs) + + # check that cleanup callback was called + speak_msg = {'type': 'speak', + 'data': {'utterance': 'I am dead', 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertTrue(self.skill.instance.my_special_var == "default") + + # check that we are not getting speak messages anymore + self.bus.emitted_msgs = [] + sleep(2) + self.assertTrue(self.bus.emitted_msgs == []) + + def test_get_response(self): + """ send "mycroft.skills.abort_question" and + confirm only get_response is aborted, speech after is still spoken""" + self.bus.emitted_msgs = [] + # skill will enter a infinite loop unless aborted + self.bus.emit(Message(f"{self.skill.skill_id}:test2.intent")) + sleep(2) + # check that intent triggered + start_msg = {'type': 'mycroft.skill.handler.start', + 'data': {'name': 'KillableSkill.handle_test_get_response_intent'}} + speak_msg = {'type': 'speak', + 'data': {'utterance': 'this is a question', + 'expect_response': True, + 'meta': {'dialog': 'question', 'data': {}, 'skill': 'abort.test'}, + 'lang': 'en-us'}} + activate_msg = {'type': 'intent.service.skills.activate', 'data': {'skill_id': 'abort.test'}} + + self.assertIn(start_msg, self.bus.emitted_msgs) + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertIn(activate_msg, self.bus.emitted_msgs) + + # check that get_response loop is aborted + # but intent continues executing + self.bus.emitted_msgs = [] + self.bus.emit(Message(f"mycroft.skills.abort_question")) + sleep(1) + + # check that stop method was NOT called + self.assertFalse(self.skill.instance.stop_called) + + # check that speak message after get_response loop was spoken + speak_msg = {'type': 'speak', + 'data': {'utterance': 'question aborted', + 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + self.assertIn(speak_msg, self.bus.emitted_msgs) + + def test_developer_stop_msg(self): + """ send "my.own.abort.msg" and confirm intent3 is aborted + send "mycroft.skills.abort_execution" and confirm intent3 ignores it""" + self.bus.emitted_msgs = [] + # skill will enter a infinite loop unless aborted + self.bus.emit(Message(f"{self.skill.skill_id}:test3.intent")) + sleep(2) + # check that intent triggered + start_msg = {'type': 'mycroft.skill.handler.start', + 'data': {'name': 'KillableSkill.handle_test_msg_intent'}} + speak_msg = {'type': 'speak', + 'data': {'utterance': "you can't abort me", + 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + self.assertIn(start_msg, self.bus.emitted_msgs) + self.assertIn(speak_msg, self.bus.emitted_msgs) + + # check that intent does NOT react to mycroft.skills.abort_execution + # developer requested a dedicated abort message + self.bus.emitted_msgs = [] + self.bus.emit(Message(f"mycroft.skills.abort_execution")) + sleep(1) + + # check that stop method was NOT called + self.assertFalse(self.skill.instance.stop_called) + + # check that intent reacts to my.own.abort.msg + self.bus.emitted_msgs = [] + self.bus.emit(Message(f"my.own.abort.msg")) + sleep(2) + + # check that stop method was called + self.assertTrue(self.skill.instance.stop_called) + + # check that TTS stop message was emmited + tts_stop = {'type': 'mycroft.audio.speech.stop', 'data': {}} + self.assertIn(tts_stop, self.bus.emitted_msgs) + + # check that cleanup callback was called + speak_msg = {'type': 'speak', + 'data': {'utterance': 'I am dead', 'expect_response': False, + 'meta': {'skill': 'abort.test'}, + 'lang': 'en-us'}} + self.assertIn(speak_msg, self.bus.emitted_msgs) + self.assertTrue(self.skill.instance.my_special_var == "default") + + # check that we are not getting speak messages anymore + self.bus.emitted_msgs = [] + sleep(2) + self.assertTrue(self.bus.emitted_msgs == [])