Skip to content

Commit

Permalink
Add hooks for start/stop STT processing
Browse files Browse the repository at this point in the history
I find it very useful to know via hook feedback when my Kalliope
actually stops *listening* (i.e. the microphone is no longer actively
recording) and when STT starts processing, especially since with some
STT providers this latter step can take some amount of time.

This alters the behaviour of the old on_start/stop_listening hooks to
fire inside the STT Utils when the listener actually stops listening and
the callback is fired, and then adds two new hooks around said callback:
on_start_stt_processing and on_stop_stt_processing. These are optional
like most others, but could be used if desired.

With this new setup, the on_start_listening/on_stop_listening only fire
when the microphone is in use by the STT library; I can't see a good
reason why this wouldn't happen, but am open to feedback if there is a
better place to put these instead.
  • Loading branch information
joshuaboniface committed Jun 5, 2023
1 parent af0af4d commit 71830d0
Show file tree
Hide file tree
Showing 6 changed files with 21 additions and 5 deletions.
2 changes: 2 additions & 0 deletions kalliope/core/ConfigurationManager/SettingLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,8 @@ def _get_hooks(settings):
"on_triggered",
"on_start_listening",
"on_stop_listening",
"on_start_stt_processing",
"on_stop_stt_processing",
"on_order_found",
"on_order_not_found",
"on_deaf",
Expand Down
10 changes: 9 additions & 1 deletion kalliope/core/HookManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ def on_start_listening(cls):
def on_stop_listening(cls):
return cls.execute_synapses_in_hook_name("on_stop_listening")

@classmethod
def on_start_stt_processing(cls):
return cls.execute_synapses_in_hook_name("on_start_stt_processing")

@classmethod
def on_stop_stt_processing(cls):
return cls.execute_synapses_in_hook_name("on_stop_stt_processing")

@classmethod
def on_order_found(cls):
return cls.execute_synapses_in_hook_name("on_order_found")
Expand Down Expand Up @@ -97,4 +105,4 @@ def execute_synapses_in_hook_name(cls, hook_name):

if isinstance(list_synapse, str):
list_synapse = [list_synapse]
return SynapseLauncher.start_synapse_by_list_name(list_synapse, new_lifo=True)
return SynapseLauncher.start_synapse_by_list_name(list_synapse, new_lifo=True)
2 changes: 0 additions & 2 deletions kalliope/core/NeuronModule.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,15 +273,13 @@ def get_audio_from_stt(callback):
Call the default STT to get an audio sample and return it into the callback method
:param callback: A callback function
"""
HookManager.on_start_listening()
# call the order listener
ol = OrderListener(callback=callback)
ol.start()
ol.join()
# wait that the STT engine has finish his job (or the neurotransmitter neuron will be killed)
if ol.stt_instance is not None:
ol.stt_instance.join()
HookManager.on_stop_listening()

def get_neuron_name(self):
"""
Expand Down
2 changes: 2 additions & 0 deletions kalliope/settings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ hooks:
on_triggered: "on-triggered-synapse"
on_start_listening:
on_stop_listening:
on_start_stt_processing:
on_stop_stt_processing:
on_order_found:
on_order_not_found: "order-not-found-synapse"
on_processed_synapses:
Expand Down
2 changes: 0 additions & 2 deletions kalliope/signals/order/order.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ def start_order_listener_thread(self):
logger.debug("[Order] Entering state: %s" % self.state)

# start listening for an order
HookManager.on_start_listening()
self.order_listener_callback_called = False
self.order_listener = OrderListener(callback=self.order_listener_callback)
self.order_listener.daemon = True
Expand All @@ -177,7 +176,6 @@ def order_listener_callback(self, order):
:type order: str
"""
logger.debug("[Order] Order listener callback called. Order to process: %s" % order)
HookManager.on_stop_listening()
self.order_to_process = order
self.order_listener_callback_called = True
# save in kalliope memory the last order
Expand Down
8 changes: 8 additions & 0 deletions kalliope/stt/Utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import logging
from kalliope import Utils, SettingLoader
from kalliope.stt import SpeechRecognizer
from kalliope.core.HookManager import HookManager
import speech_recognition as sr

logging.basicConfig()
Expand Down Expand Up @@ -39,11 +40,18 @@ def run(self):
Start the thread that listen the microphone and then give the audio to the callback method
"""
if self.audio_stream is None:
HookManager.on_start_listening()
Utils.print_success("Say something!")
with self.microphone as source:
self.audio_stream = self.recognizer.listen(source)
HookManager.on_stop_listening()
Utils.print_success("Finished listening!")

HookManager.on_start_stt_processing()
Utils.print_success("Processing via STT engine!")
self.callback(self.recognizer, self.audio_stream)
HookManager.on_stop_stt_processing()
Utils.print_success("Finished processing!")

def start_processing(self):
"""
Expand Down

0 comments on commit 71830d0

Please sign in to comment.