Merge pull request #792 from stanford-oval/wip/volume-control

Extend the audio control protocol with the missing features

.gitignore

@@ -30,6 +30,7 @@ po/*.json
 *.tgz
 *.tt.json
 data/builtins/thingengine.builtin/*.tt
+data/builtins/org.thingpedia.builtin.thingengine.builtin/*.tt
 lib/engine/devices/builtins/faq.json
 lib/engine/db/sqlite/schema.json
 

Makefile

@@ -40,11 +40,12 @@ generated := \
 	$(generated_early) \
 	$(patsubst %.po,%.mo,$(wildcard po/*.po)) \
 	$(built_bundled_templates) \
-	data/builtins/thingengine.builtin/dataset.tt \
+	data/builtins/org.thingpedia.builtin.thingengine.builtin/dataset.tt \
 	lib/engine/db/sqlite/schema.json \
-	lib/engine/devices/builtins/test.tt.json \
-	lib/engine/devices/builtins/thingengine.tt.json \
-	lib/engine/devices/builtins/thingengine.builtin.tt.json \
+	lib/engine/devices/builtins/org.thingpedia.builtin.test.tt.json \
+	lib/engine/devices/builtins/org.thingpedia.builtin.thingengine.tt.json \
+	lib/engine/devices/builtins/org.thingpedia.builtin.thingengine.builtin.tt.json \
+	lib/engine/devices/builtins/org.thingpedia.volume-control.tt.json \
 	lib/engine/devices/builtins/faq.json
 
 $(built_bundled_templates) : $(template_sources) lib/sentence-generator/compiler/*.ts $(generated_early)
@@ -67,8 +68,8 @@ lib/engine/devices/builtins/%.tt.json : data/builtins/%/manifest.tt
 	node -e 'console.log(JSON.stringify(require("fs").readFileSync(process.argv[1]).toString("utf8")))' $< > $@.tmp
 	mv $@.tmp $@
 
-data/builtins/thingengine.builtin/dataset.tt data/builtins/thingengine.builtin/manifest.tt lib/engine/devices/builtins/faq.json &: data/builtins/thingengine.builtin/dataset.tt.in data/builtins/thingengine.builtin/manifest.tt.in data/builtins/thingengine.builtin/faq.yaml
-	ts-node data/builtins/thingengine.builtin/merge-faq
+data/builtins/org.thingpedia.builtin.thingengine.builtin/dataset.tt data/builtins/org.thingpedia.builtin.thingengine.builtin/manifest.tt lib/engine/devices/builtins/faq.json &: data/builtins/org.thingpedia.builtin.thingengine.builtin/dataset.tt.in data/builtins/org.thingpedia.builtin.thingengine.builtin/manifest.tt.in data/builtins/org.thingpedia.builtin.thingengine.builtin/faq.yaml
+	ts-node data/builtins/org.thingpedia.builtin.thingengine.builtin/merge-faq
 
 %.mo : %.po
 	msgfmt $< -o $@

data/builtins/thingengine.builtin/faq.yaml → data/builtins/org.thingpedia.builtin.thingengine.builtin/faq.yaml

@@ -16,8 +16,7 @@ about_almond_age:
     - How old are you?
     - When were you born?
   a:
-    - Officially, I'm beta. The plan is for me to be released in the first half of 2021.
-    - I was born in 2015 in a class at Stanford. The plan is for me to be released in the first half of 2021.
+    - I was born in 2015 in a class at Stanford. The plan is for me to be released in the second half of 2021.
 
 about_almond_how_are_you:
   q:
@@ -26,14 +25,17 @@ about_almond_how_are_you:
     - How are {you|u} today?
     - How are {you|u} feeling today?
   a:
-    - Great. Thanks for asking. How can I help you?
+    - Great, thanks for asking! How can I help you?
 
 about_almond_location:
   q:
     - Where are you?
     - Where are you now?
   a:
+    cloud:
     - My creators are mostly at Stanford. I'm an open-source assistant in the cloud.
+    default:
+    - My creators are mostly at Stanford, but I'm right here in your device.
 
 about_almond_comparison:
   q:
@@ -101,7 +103,7 @@ general_help:
     - Can you give me another suggestion?
     - Show skills
   a:
-    - I'm a new virtual assistant. At the moment, I can help you with music, jokes, and managing some IoT devices.
+    - I'm a new virtual assistant. At the moment, I can help you with music, jokes, news, podcasts, weather, reminders, and managing IoT devices.
 
 iot_help:
   q:
@@ -141,8 +143,8 @@ positive_emotion_cool:
 
 abusive:
   q:
-    - Kill yourself
     - You suck
+    - Kill yourself
     - Go die
     - Fuck you
   a:
@@ -210,9 +212,9 @@ how_to_use_wakeword:
     - How do I enable audio?
   a:
     cloud:
-    - I am sorry, continuous voice input with wake word detection is not available in the web version of Genie.
+    - To speak to me in voice using a wake word, you will need the Genie client running in a device with speaker and microphone. Then you can just say “Hey Genie” to wake me up. You will hear a sound when the wake word is recognized correctly.
     default:
-    - To speak to me in voice using a wake word, first make sure that voice input is enabled in Settings. Then you can just say “Computer” to wake me up. You will hear a sound when the wake word is recognized correctly.
+    - To speak to me in voice using a wake word, first make sure that voice input is enabled in Settings. Then you can just say “Hey Genie” to wake me up. You will hear a sound when the wake word is recognized correctly.
 
 troubleshooting_home_assistant:
   q:

data/builtins/thingengine.builtin/manifest.tt.in → data/builtins/org.thingpedia.builtin.thingengine.builtin/manifest.tt.in

@@ -1,4 +1,4 @@
-class @org.thingpedia.builtin.thingengine.builtin
+class @org.thingpedia.builtin.thingengine.builtin extends @org.thingpedia.volume-control
 #_[thingpedia_name="Miscellaneous Interfaces"]
 #_[thingpedia_description="Time, random numbers, and other commands not specific to any skill."]
 #_[name="Miscellaneous Interfaces"]
@@ -370,13 +370,29 @@ class @org.thingpedia.builtin.thingengine.builtin
   #[doc="change the user's preferred temperature unit (a setting)"]
   #[confirm=false];
 
+  action stop()
+  #_[canonical=["stop audio",
+                "stop playing",
+                "stop playback"]]
+  #[doc="pause the currently playing skill"]
+  #[confirm=false];
+
   action pause()
-  #_[canonical=["pause", "pause audio", "stop audio"]]
+  #_[canonical=["pause",
+                "pause audio",
+                "pause playback",
+                "pause playing"]]
   #[doc="pause the currently playing skill"]
   #[confirm=false];
 
   action resume()
-  #_[canonical=["resume", "resume audio"]]
+  #_[canonical=["resume",
+                "resume audio",
+                "resume playback",
+                "resume playing",
+                "start playing again",
+                "continue playback",
+                "continue playing"]]
   #[doc="resume the currently playing skill"]
   #[confirm=false]
   #_[on_error={

data/builtins/org.thingpedia.volume-control/dataset.tt

@@ -0,0 +1,33 @@
+dataset @org.thingpedia.volume-control language "en" {
+
+  action (p_name : String) := @org.thingpedia.volume-control(name=p_name).raise_volume()
+  #_[utterances=["increase the volume of the ${p_name:const} device",
+                 "increase my ${p_name:const} device 's volume",
+                 "make the ${p_name:const} device louder"]];
+
+  action (p_name : String) := @org.thingpedia.volume-control(name=p_name).lower_volume()
+  #_[utterances=["decrease the volume of my ${p_name:const} device",
+                 "decrease my ${p_name:const} device 's volume",
+                 "reduce the volume of the ${p_name:const} device",
+                 "reduce my ${p_name:const} device 's volume",
+                 "make the ${p_name:const} device softer"]];
+
+  action (p_name : String) := @org.thingpedia.volume-control(name=p_name).mute()
+  #_[utterances=["mute my ${p_name:const} device",
+                 "silence my ${p_name:const} device 's volume",
+                 "turn off the volume of the ${p_name:const} device",
+                 "switch off the sound for my ${p_name:const} device",
+                 "mute my ${p_name:const} device 's sound",
+                 "make my ${p_name:const} device silent"]];
+
+  action (p_name : String) := @org.thingpedia.volume-control(name=p_name).unmute()
+  #_[utterances=["unmute my ${p_name:const} device",
+                 "unmute the ${p_name:const} device 's volume",
+                 "unmute my ${p_name:const} device 's sound",
+                 "turn on the ${p_name:const} device sound",
+                 "stop muting my ${p_name:const} device"]];
+
+  action (p_name : String, p_volume : Number) := @org.thingpedia.volume-control(name=p_name).set_volume(volume=p_volume)
+  #_[utterances=["turn my ${p_name:const} device sound to ${p_volume:const}",
+                 "set the ${p_name:const} device 's volume to ${p_volume:const}"]];
+}

data/builtins/org.thingpedia.volume-control/manifest.tt

@@ -0,0 +1,55 @@
+abstract class @org.thingpedia.volume-control
+#_[thingpedia_name="Volume Control"]
+#_[thingpedia_description="Interface for Volume Control."]
+#_[description="Interface for Volume Control."]
+#[license="Apache-2.0"]
+#[license_gplcompatible=true]
+#[subcategory="home"]
+{
+
+  action raise_volume()
+  #_[canonical=["increase the volume of the speaker",
+                "increase my speaker 's volume",
+                "make the speaker louder",
+                "increase the speaker volume"]]
+  #[confirm=false]
+  #[doc="increase the volume of the speaker"];
+
+  action lower_volume()
+  #_[canonical=["decrease the volume of my speaker",
+                "decrease my speaker 's volume",
+                "reduce the volume of the speaker",
+                "reduce my speaker 's volume",
+                "make the speaker softer"]]
+  #[confirm=false]
+  #[doc="decrease the volume of the speaker"];
+
+  action mute()
+  #_[canonical=["mute my speaker",
+                "silence my speaker 's volume",
+                "turn off the volume of the speaker",
+                "switch off the sound for my speaker",
+                "mute my speaker 's sound",
+                "make my speaker silent"]]
+  #[confirm=false]
+  #[doc="mute the speaker"];
+
+  action unmute()
+  #_[canonical=["unmute my speaker",
+                "unmute the speaker 's volume",
+                "unmute my speaker 's sound",
+                "turn on the speaker sound",
+                "stop muting my speaker"]]
+  #[confirm=false]
+  #[doc="unmute the speaker"];
+
+  action set_volume(in req volume: Number #_[prompt="from 0 to 100, what volume do you want to set"]
+                    #_[canonical={
+                      base=["volume", "value"],
+                      preposition=["to #"]
+                    }])
+  #_[canonical="set the volume of the speaker"]
+  #[confirm=false]
+  #[doc="set the speaker volume"];
+
+}

lib/dialogue-agent/audio/controller.ts

@@ -121,7 +121,7 @@ export default class AudioController extends events.EventEmitter {
      *
      * @param conversationId
      */
-    getPlayer(conversationId : string) : AudioPlayer|undefined {
+    getPlayer(conversationId ?: string) : AudioPlayer|undefined {
         const state = this._getPlayer(conversationId);
         return state?.player;
     }
@@ -136,14 +136,14 @@ export default class AudioController extends events.EventEmitter {
      */
     async resumeAudio(conversationId ?: string) {
         const state = this._getPlayer(conversationId);
-        if (!state || !state.iface)
+        if (!state)
             throw new CustomError(`no_device_playing`, `No interface registered to resume audio`);
 
-        if (!state.iface.resume)
-            throw new CustomError(`unsupported`, `Resuming is not supported`);
-
         state.timestamp = Date.now();
-        await state.iface.resume(state.player.conversationId);
+        if (state.iface && state.iface.resume)
+            await state.iface.resume(state.player.conversationId);
+        else
+            await state.player.resume();
     }
 
     private _normalizeCompatIface(iface : AudioDevice|(() => Promise<void>)) {
@@ -233,6 +233,7 @@ export default class AudioController extends events.EventEmitter {
         if (state.iface)
             await state.iface.stop(state.player.conversationId);
         state.device = device;
+        state.iface = null;
         console.log(`Switching audio to ${state.device.uniqueId}`);
         await state.player.playURLs(urls);
     }
@@ -282,6 +283,49 @@ export default class AudioController extends events.EventEmitter {
         }
     }
 
+    /**
+     * Pause all audio coming from this assistant.
+     *
+     * This method will inform the currently playing device that it must pause
+     * playing. It corresponds to the command "pause".
+     *
+     * This command differs from {@link stopAudio} because the audio device
+     * should interpret this request in a way that makes the audio resumable.
+     *
+     * @param the conversation ID associated with the command; if specified,
+     *   only audio associated with that conversation will be paused
+     */
+    async pauseAudio(conversationId ?: string) {
+        this.emit('stop', conversationId);
+
+        if (conversationId !== undefined) {
+            const state = this._getPlayer(conversationId);
+            if (!state)
+                return;
+
+            await state.player.pause();
+            if (state.iface !== null) {
+                if (state.iface.pause)
+                    await state.iface.pause(state.player.conversationId);
+                else
+                    await state.iface.stop(state.player.conversationId);
+            }
+
+            // the interface continues to be the current audio interface
+            // so it will be resumed on a resume command later
+        } else {
+            for (const state of this._players.values()) {
+                await state.player.pause();
+                if (state.iface !== null) {
+                    if (state.iface.pause)
+                        await state.iface.pause(state.player.conversationId);
+                    else
+                        await state.iface.stop(state.player.conversationId);
+                }
+            }
+        }
+    }
+
     /**
      * Stop all audio coming from this assistant.
      *

lib/dialogue-agent/audio/interface.ts

@@ -29,6 +29,11 @@
      */
     stop(conversationId : string) : void;
 
+    /**
+     * Pause all playback.
+     */
+    pause?(conversationId : string) : void;
+
     /**
      * Resume playback.
      */
@@ -119,6 +124,30 @@ export interface AudioPlayer {
      */
     stop() : Promise<void>;
 
+    /**
+     * Pause playing.
+     *
+     * This method should pause all playback, while preserving the state
+     * of all queues so playback can be resumed.
+     *
+     * Depending on the kind of backend currently playing, this might have
+     * no effect, as backends might rely on the skill to invoke third-party APIs
+     * to stop.
+     */
+    pause() : Promise<void>;
+
+    /**
+     * Resume playing.
+     *
+     * This method should attempt to resume playback. It should return an
+     * error if resuming is not possible for any reason.
+     *
+     * Depending on the kind of backend currently playing, this might have
+     * no effect, as backends might rely on the skill to invoke third-party APIs
+     * to stop.
+     */
+    resume() : Promise<void>;
+
     /**
      * Start playing with the given URLs.
      *
@@ -132,14 +161,38 @@ export interface AudioPlayer {
     playURLs(urls : string[]) : Promise<void>;
 
     /**
-     * Set the output volume.
+     * Set the output volume to a specific value.
      *
      * @param volume the volume, between 0 and 100
      */
     setVolume(volume : number) : Promise<void>;
 
+    /**
+     * Adjust the output volume by the given delta.
+     *
+     * @param delta the volume delta, between -100 and 100
+     */
+    adjustVolume(delta : number) : Promise<void>;
+
     /**
      * Mute or unmute the audio.
      */
     setMute(mute : boolean) : Promise<void>;
+
+    /**
+     * Enable or disable voice input.
+     *
+     * This command only affects wake-word activation. If the assistant can
+     * be activated by other means (e.g. a button) it is not affected by enabling
+     * the wake word.
+     */
+    setVoiceInput(input : boolean) : Promise<void>;
+
+    /**
+     * Enable or disable voice output.
+     *
+     * This command only affects speech from the agent. It does not affect
+     * alert sounds or background audio.
+     */
+    setVoiceOutput(output : boolean) : Promise<void>;
 }