/
voice-assistant-prototype.yaml
421 lines (364 loc) · 14.6 KB
/
voice-assistant-prototype.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
substitutions:
# Phases of the Voice Assistant
# IDLE: The voice assistant is ready to be triggered by a wake-word
voice_assist_idle_phase_id: '1'
# LISTENING: The voice assistant is ready to listen to a voice command (after being triggered by the wake word)
voice_assist_listening_phase_id: '2'
# THINKING: The voice assistant is currently processing the command
voice_assist_thinking_phase_id: '3'
# REPLYING: The voice assistant is replying to the command
voice_assist_replying_phase_id: '4'
# NOT_READY: The voice assistant is not ready
voice_assist_not_ready_phase_id: '10'
# ERROR: The voice assistant encountered an error
voice_assist_error_phase_id: '11'
# MUTED: The voice assistant is muted and will not reply to a wake-word
voice_assist_muted_phase_id: '12'
globals:
# Global initialisation variable. Initialized to true and set to false once everything is connected. Only used to have a smooth "plugging" experience
- id: init_in_progress
type: bool
restore_value: no
initial_value: 'true'
# Global variable tracking the phase of the voice assistant (defined above). Initialized to not_ready
- id: voice_assistant_phase
type: int
restore_value: no
initial_value: ${voice_assist_not_ready_phase_id}
# Declaration of our LED, with two effects: A "Slow Pulse" and a "Fast Pulse" that will be used as feedback for the different phases of our voice assistant
light:
- platform: esp32_rmt_led_strip
rgb_order: GRB
pin: GPIO15
num_leds: 1
rmt_channel: 0
chipset: WS2812
name: "Status LED"
id: led
disabled_by_default: True
entity_category: diagnostic
icon: mdi:led-on
default_transition_length: 0s
effects:
- pulse:
name: "Slow Pulse"
transition_length: 250ms
update_interval: 250ms
min_brightness: 50%
max_brightness: 100%
- pulse:
name: "Fast Pulse"
transition_length: 100ms
update_interval: 100ms
min_brightness: 50%
max_brightness: 100%
script:
# Master script controlling the LED, based on different conditions: initialization in progress, wifi and API connected, and the current voice assistant phase.
# For the sake of simplicity and re-usability, the script calls child scripts defined below.
# This script will be called every time one of these conditions is changing.
- id: control_led
then:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- if:
condition:
wifi.connected:
then:
- if:
condition:
api.connected:
then:
- lambda: |
switch(id(voice_assistant_phase)) {
case ${voice_assist_listening_phase_id}:
id(control_led_voice_assist_listening_phase).execute();
break;
case ${voice_assist_thinking_phase_id}:
id(control_led_voice_assist_thinking_phase).execute();
break;
case ${voice_assist_replying_phase_id}:
id(control_led_voice_assist_replying_phase).execute();
break;
case ${voice_assist_error_phase_id}:
id(control_led_voice_assist_error_phase).execute();
break;
case ${voice_assist_muted_phase_id}:
id(control_led_voice_assist_muted_phase).execute();
break;
case ${voice_assist_not_ready_phase_id}:
id(control_led_voice_assist_not_ready_phase).execute();
break;
default:
id(control_led_voice_assist_idle_phase).execute();
break;
}
else:
- script.execute: control_led_no_ha_connection_state
else:
- script.execute: control_led_no_ha_connection_state
else:
- script.execute: control_led_init_state
# Script executed during initialisation: In this example: Turn the LED in green with a slow pulse 🟢
- id: control_led_init_state
then:
- light.turn_on:
id: led
blue: 0%
red: 0%
green: 100%
effect: "Fast Pulse"
# Script executed when the device has no connection to Home Assistant: In this example: Turn off the LED
- id: control_led_no_ha_connection_state
then:
- light.turn_off:
id: led
# Script executed when the voice assistant is idle (waiting for a wake word): In this example: Turn the LED in white with 20% of brightness ⚪
- id: control_led_voice_assist_idle_phase
then:
- light.turn_on:
id: led
blue: 100%
red: 100%
green: 100%
brightness: 20%
effect: "none"
# Script executed when the voice assistant is listening to a command: In this example: Turn the LED in blue with a slow pulse 🔵
- id: control_led_voice_assist_listening_phase
then:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
effect: "Slow Pulse"
# Script executed when the voice assistant is processing the command: In this example: Turn the LED in blue with a fast pulse 🔵
- id: control_led_voice_assist_thinking_phase
then:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
effect: "Fast Pulse"
# Script executed when the voice assistant is replying to a command: In this example: Turn the LED in blue, solid (no pulse) 🔵
- id: control_led_voice_assist_replying_phase
then:
- light.turn_on:
id: led
blue: 100%
red: 0%
green: 0%
brightness: 100%
effect: "none"
# Script executed when the voice assistant encounters an error: In this example: Turn the LED in red, solid (no pulse) 🔴
- id: control_led_voice_assist_error_phase
then:
- light.turn_on:
id: led
blue: 0%
red: 100%
green: 0%
brightness: 100%
effect: "none"
# Script executed when the voice assistant is muted: In this example: Turn off the LED
- id: control_led_voice_assist_muted_phase
then:
- light.turn_off:
id: led
# Script executed when the voice assistant is not ready: In this example: Turn off the LED
- id: control_led_voice_assist_not_ready_phase
then:
- light.turn_off:
id: led
esphome:
name: voice-assistant-prototype
friendly_name: Voice Assistant Prototype
# Automation to perform every time the device boots
on_boot:
priority: 600
then:
# Run the script to refresh the LED status
- script.execute: control_led
# If after 30 seconds, the device is still initializing (It did not yet connect to Home Assistant), turn off the init_in_progress variable and run the script to refresh the LED status
- delay: 30s
- if:
condition:
lambda: return id(init_in_progress);
then:
- lambda: id(init_in_progress) = false;
- script.execute: control_led
esp32:
board: esp32dev
# This is important. ESPHome supports two frameworks: Arduino and ESP-IDF. ESP-IDF is needed to include an audio library called ESP_ADF used in our voice assistant
framework:
type: esp-idf
# Enable logging
logger:
# Enable Home Assistant API
api:
encryption:
# You may need to enter this encryption key in Home Assistant when you integrate the device for the first time
key: !secret tutorial-voice-assistant-encryption-key
# If the device connects, or disconnects, to Home Assistant: Run the script to refresh the LED status
on_client_connected:
- script.execute: control_led
on_client_disconnected:
- script.execute: control_led
ota:
password: !secret tutorial-voice-assistant-ota-password
wifi:
ssid: !secret wifi_ssid
password: !secret wifi_password
# If the device connects, or disconnects, to the Wifi: Run the script to refresh the LED status
on_connect:
- script.execute: control_led
on_disconnect:
- script.execute: control_led
# Enable fallback hotspot (captive portal) in case wifi connection fails
ap:
ssid: !secret tutorial-voice-assistant-hotspot-ssid
password: !secret tutorial-voice-assistant-hotspot-password
# This is how to include the Espressif Audio Development Framework.
# This is needed to be able to use VAD (Voice audio detection) and prevent the voice assistant from being constantly streaming audio to Home Assistant
# For now, this component is not documented, nor on the code base of ESPHome, hence the reference to the external component.
esp_adf:
external_components:
- source: github://pr#5230
components:
- esp_adf
refresh: 0s
captive_portal:
# Declaration of the switch that will be used to turn on or off (mute) or voice assistant
switch:
- platform: template
name: Enable Voice Assistant
id: use_wake_word
optimistic: true
restore_mode: RESTORE_DEFAULT_ON
icon: mdi:assistant
# When the switch is turned on (on Home Assistant):
# Start the voice assistant component
# Set the correct phase and run the script to refresh the LED status
on_turn_on:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
- if:
condition:
not:
- voice_assistant.is_running
then:
- voice_assistant.start_continuous
- script.execute: control_led
# When the switch is turned off (on Home Assistant):
# Stop the voice assistant component
# Set the correct phase and run the script to refresh the LED status
on_turn_off:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- voice_assistant.stop
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- script.execute: control_led
# This is our two i2s buses with the correct pins.
# You can refer to the wirinng diagram of our voice assistant for more details
i2s_audio:
- id: i2s_in
i2s_lrclk_pin: GPIO25
i2s_bclk_pin: GPIO26
- id: i2s_out
i2s_lrclk_pin: GPIO12
i2s_bclk_pin: GPIO27
# This is the declaration of our microphone.
# It includes the data pin (You can refer to the wiring diagram of our voice assistant for more details)
# It references the correct i2s bus declared above.
microphone:
platform: i2s_audio
id: external_microphone
adc_type: external
i2s_audio_id: i2s_in
i2s_din_pin: GPIO34
pdm: false
# This is the declaration of our speaker.
# It includes the data pin (You can refer to the wiring diagram of our voice assistant for more details)
# It references the correct i2s bus declared above.
speaker:
platform: i2s_audio
id: external_speaker
dac_type: external
i2s_audio_id: i2s_out
i2s_dout_pin: GPIO33
# This is the declaration of our voice assistant
# It references the microphone and speaker declared above.
voice_assistant:
id: va
microphone: external_microphone
speaker: external_speaker
use_wake_word: true
# This is how I personally tune my voice assistant, you may have to test a few values for the 4 parameters above
noise_suppression_level: 4
auto_gain: 31dBFS
volume_multiplier: 8.0
vad_threshold: 3
# When the voice assistant connects to HA:
# Set init_in_progress to false (Initialization is over).
# If the switch is on, start the voice assistant
# In any case: Set the correct phase and run the script to refresh the LED status
on_client_connected:
- lambda: id(init_in_progress) = false;
- if:
condition:
switch.is_on: use_wake_word
then:
- voice_assistant.start_continuous:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
else:
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- script.execute: control_led
# When the voice assistant disconnects to HA:
# Stop the voice assistant
# Set the correct phase and run the script to refresh the LED status
on_client_disconnected:
- lambda: id(voice_assistant_phase) = ${voice_assist_not_ready_phase_id};
- voice_assistant.stop
- script.execute: control_led
# When the voice assistant starts to listen: Set the correct phase and run the script to refresh the LED status
on_listening:
- lambda: id(voice_assistant_phase) = ${voice_assist_listening_phase_id};
- script.execute: control_led
# When the voice assistant starts to think: Set the correct phase and run the script to refresh the LED status
on_stt_vad_end:
- lambda: id(voice_assistant_phase) = ${voice_assist_thinking_phase_id};
- script.execute: control_led
# When the voice assistant starts to reply: Set the correct phase and run the script to refresh the LED status
on_tts_stream_start:
- lambda: id(voice_assistant_phase) = ${voice_assist_replying_phase_id};
- script.execute: control_led
# When the voice assistant finished to reply: Set the correct phase and run the script to refresh the LED status
on_tts_stream_end:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
- script.execute: control_led
# When the voice assistant encounters an error:
# Set the error phase and run the script to refresh the LED status
# Wait 1 second and set the correct phase (idle or muted depending on the state of the switch) and run the script to refresh the LED status
on_error:
- if:
condition:
lambda: return !id(init_in_progress);
then:
- lambda: id(voice_assistant_phase) = ${voice_assist_error_phase_id};
- script.execute: control_led
- delay: 1s
- if:
condition:
switch.is_on: use_wake_word
then:
- lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id};
else:
- lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};
- script.execute: control_led