onju-voice-microwakeword.yaml

substitutions:
  name: "onju-voice"
  friendly_name: "Onju Voice Satellite"
  project_version: "1.0.0"
  device_description: "Onju Voice Satellite with ESPHome software and microWakeWord"
external_components:
  - source:
      type: git
      url: https://github.com/gnumpi/esphome_audio
      ref: main
    components: [ adf_pipeline, i2s_audio ]

esphome:
  name: "${name}"
  friendly_name: "${friendly_name}"
  comment: "${device_description}"
  name_add_mac_suffix: true
  project:
    name: tetele.onju_voice_satellite
    version: "${project_version}"
  min_version: 2024.3.0
  platformio_options:
    board_build.flash_mode: dio
    build_flags: "-DBOARD_HAS_PSRAM"
    board_build.arduino.memory_type: qio_opi
  on_boot:
    then:
      - light.turn_on:
          id: top_led
          effect: slow_pulse
          red: 100%
          green: 60%
          blue: 0%
      - wait_until:
          condition:
            wifi.connected:
      - light.turn_on:
          id: top_led
          effect: pulse
          red: 0%
          green: 100%
          blue: 0%
      - wait_until:
          condition:
            api.connected:
      - light.turn_on:
          id: top_led
          effect: none
          red: 0%
          green: 100%
          blue: 0%
      - delay: 1s
      - script.execute: reset_led

dashboard_import:
  package_import_url: github://tetele/onju-voice-satellite/esphome/onju-voice-microwakeword.yaml@main

esp32:
  board: esp32-s3-devkitc-1
  framework:
    type: esp-idf
    version: recommended
    sdkconfig_options:
      # need to set a s3 compatible board for the adf-sdk to compile
      # board specific code is not used though
      CONFIG_ESP32_S3_BOX_BOARD: "y"

psram:
  mode: octal
  speed: 80MHz

# Enable logging
logger:

# Allow OTA updates
ota:

# Allow provisioning Wi-Fi via serial
improv_serial:

wifi:
  # Set up a wifi access point
  ap:
    ssid: "${friendly_name}"

# In combination with the `ap` this allows the user
# to provision wifi credentials to the device via WiFi AP.
captive_portal:

api:
  services:
    - service: start_va
      then:
        - voice_assistant.start
    - service: stop_va
      then:
        - voice_assistant.stop
    - service: notification_on
      then:
        - script.execute: turn_on_notification
    - service: notification_clear
      then:
        - script.execute: clear_notification

globals:
  - id: thresh_percent
    type: float
    initial_value: "0.03"
    restore_value: false
  - id: touch_calibration_values_left
    type: uint32_t[5]
    restore_value: false
  - id: touch_calibration_values_center
    type: uint32_t[5]
    restore_value: false
  - id: touch_calibration_values_right
    type: uint32_t[5]
    restore_value: false
  - id: notification
    type: bool
    restore_value: false

interval:
  - interval: 1s
    then:
      - script.execute:
          id: calibrate_touch
          button: 0
      - script.execute:
          id: calibrate_touch
          button: 1
      - script.execute:
          id: calibrate_touch
          button: 2


i2s_audio:
  - id: i2s_shared
    i2s_lrclk_pin: GPIO13
    i2s_bclk_pin: GPIO18
    access_mode: duplex


adf_pipeline:
  - platform: i2s_audio
    type: audio_out
    id: adf_i2s_out
    i2s_audio_id: i2s_shared
    i2s_dout_pin: GPIO12
    sample_rate: 16000
    adf_alc: true
    bits_per_sample: 32bit
    fixed_settings: true

  - platform: i2s_audio
    type: audio_in
    id: adf_i2s_in
    i2s_audio_id: i2s_shared
    i2s_din_pin: GPIO17
    channel: right
    pdm: false
    sample_rate: 16000
    bits_per_sample: 32bit
    fixed_settings: true


microphone:
  - platform: adf_pipeline
    id: onju_microphone
    keep_pipeline_alive: true
    gain_log2: 3
    pipeline:
      - adf_i2s_in
      - self

media_player:
  - platform: adf_pipeline
    id: onju_out
    name: None
    internal: false
    keep_pipeline_alive: true
    pipeline:
      - self
      - resampler
      - adf_i2s_out
    on_state: 
      then:
        - lambda: |-
            static float old_volume = -1;
            float new_volume = id(onju_out).volume;
            if(abs(new_volume-old_volume) > 0.0001) {
              if(old_volume != -1) {
                id(show_volume)->execute();
              }
            }
            old_volume = new_volume;

micro_wake_word:
  model: okay_nabu
  # model: hey_jarvis
  # model: alexa
  on_wake_word_detected:
    - if:
        condition: media_player.is_playing
        then:
          - media_player.pause
    - voice_assistant.start:
        wake_word: !lambda return wake_word;

voice_assistant:
  id: va
  microphone: onju_microphone
  media_player: onju_out
  use_wake_word: false
  on_listening:
    - light.turn_on:
        id: top_led
        blue: 100%
        red: 100%
        green: 100%
        brightness: 100%
        effect: listening
  on_stt_vad_end:
    - light.turn_on:
        id: top_led
        blue: 100%
        red: 0%
        green: 20%
        brightness: 70%
        effect: processing
  on_tts_end:
    - light.turn_on:
        id: top_led
        blue: 0%
        red: 20%
        green: 100%
        effect: speaking
  on_end:
    - delay: 200ms
    - wait_until:
        not:
          media_player.is_playing: onju_out
    - script.execute: reset_led
    - if:
        condition:
          and:
            - switch.is_on: use_wake_word
            - binary_sensor.is_off: mute_switch
        then:
          - delay: 200ms
          - micro_wake_word.start
  on_client_connected:
    - if:
        condition:
          and:
            - switch.is_on: use_wake_word
            - binary_sensor.is_off: mute_switch
        then:
          - micro_wake_word.start:
  on_client_disconnected:
    - if:
        condition:
          and:
            - switch.is_on: use_wake_word
            - binary_sensor.is_off: mute_switch
        then:
          - voice_assistant.stop:
          - micro_wake_word.stop:
  on_error:
    - light.turn_on:
        id: top_led
        blue: 0%
        red: 100%
        green: 0%
        effect: none
    - delay: 1s
    - script.execute: reset_led

number:
  - platform: template
    name: "Touch threshold percentage"
    id: touch_threshold_percentage
    update_interval: never
    entity_category: config
    initial_value: 0.75
    min_value: 0.25
    max_value: 5
    step: 0.05
    optimistic: true
    on_value:
      then:
        - lambda: !lambda |-
            id(thresh_percent) = 0.01 * x;

esp32_touch:
  setup_mode: false
  sleep_duration: 2ms
  measurement_duration: 800us
  low_voltage_reference: 0.8V
  high_voltage_reference: 2.4V

  filter_mode: IIR_16
  debounce_count: 2
  noise_threshold: 0
  jitter_step: 0
  smooth_mode: IIR_2

  denoise_grade: BIT8
  denoise_cap_level: L0

binary_sensor:
  - platform: esp32_touch
    id: volume_down
    pin: GPIO4
    threshold: 539000
    on_press:
      then:
        - light.turn_on: left_led
        - script.execute:
            id: set_volume
            volume: -0.05
        - delay: 750ms
        - while:
            condition:
              binary_sensor.is_on: volume_down
            then:
              - script.execute:
                  id: set_volume
                  volume: -0.05
              - delay: 150ms
    on_release:
      then:
        - light.turn_off: left_led

  - platform: esp32_touch
    id: volume_up
    pin: GPIO2
    threshold: 580000
    on_press:
      then:
        - light.turn_on: right_led
        - script.execute:
            id: set_volume
            volume: 0.05
        - delay: 750ms
        - while:
            condition:
              binary_sensor.is_on: volume_up
            then:
              - script.execute:
                  id: set_volume
                  volume: 0.05
              - delay: 150ms
    on_release:
      then:
        - light.turn_off: right_led

  - platform: esp32_touch
    id: action
    pin: GPIO3
    threshold: 751000
    on_click:
      - if:
          condition:
            or:
              - switch.is_off: use_wake_word
              - binary_sensor.is_on: mute_switch
          then:
            - logger.log:
                tag: "action_click"
                format: "Voice assistant is running: %s"
                args: ['id(va).is_running() ? "yes" : "no"']
            - if:
                condition: media_player.is_playing
                then:
                  - media_player.stop
            - if:
                condition: voice_assistant.is_running
                then:
                  - voice_assistant.stop:
                else:
                  - voice_assistant.start:
          else:
            - logger.log:
                tag: "action_click"
                format: "Voice assistant was running with wake word detection enabled. Starting continuously"
            - if:
                condition: media_player.is_playing
                then:
                  - media_player.stop
            - voice_assistant.stop
            - delay: 1s
            - script.execute: reset_led
            - script.wait: reset_led
            - voice_assistant.start_continuous:

  - platform: gpio
    id: mute_switch
    pin:
      number: GPIO38
      mode: INPUT_PULLUP
    name: Disable wake word
    on_press:
      - script.execute: turn_off_wake_word
    on_release:
      - script.execute: turn_on_wake_word

light:
  - platform: esp32_rmt_led_strip
    id: leds
    pin: GPIO11
    chipset: SK6812
    num_leds: 6
    rgb_order: grb
    rmt_channel: 0
    default_transition_length: 0s
    gamma_correct: 2.8
  - platform: partition
    id: left_led
    segments:
      - id: leds
        from: 0
        to: 0
    default_transition_length: 100ms
  - platform: partition
    id: top_led
    segments:
      - id: leds
        from: 1
        to: 4
    default_transition_length: 100ms
    effects:
      - pulse:
          name: pulse
          transition_length: 250ms
          update_interval: 250ms
      - pulse:
          name: slow_pulse
          transition_length: 1s
          update_interval: 2s
      - addressable_lambda:
          name: show_volume
          update_interval: 50ms
          lambda: |-
            int int_volume = int(id(onju_out).volume * 100.0f * it.size());
            int full_leds = int_volume / 100;
            int last_brightness = int_volume % 100;
            int i = 0;
            for(; i < full_leds; i++) {
              it[i] = Color::WHITE;
            }
            if(i < 4) {
              it[i++] = Color(64, 64, 64).fade_to_white(last_brightness*256/100);
            }
            for(; i < it.size(); i++) {
              it[i] = Color(64, 64, 64);
            }
      - addressable_twinkle:
          name: listening_ww
          twinkle_probability: 1%
      - addressable_twinkle:
          name: listening
          twinkle_probability: 45%
      - addressable_scan:
          name: processing
          move_interval: 80ms
      - addressable_flicker:
          name: speaking
          intensity: 35%
  - platform: partition
    id: right_led
    segments:
      - id: leds
        from: 5
        to: 5
    default_transition_length: 100ms

script:
  - id: reset_led
    then:
      - if:
          condition:
            - lambda: return id(notification);
          then:
            - light.turn_on:
                id: top_led
                blue: 100%
                red: 100%
                green: 0%
                brightness: 100%
                effect: slow_pulse
          else:
            - if:
                condition:
                  and:
                    - switch.is_on: use_wake_word
                    - binary_sensor.is_off: mute_switch
                then:
                  - light.turn_on:
                      id: top_led
                      blue: 100%
                      red: 100%
                      green: 0%
                      brightness: 60%
                      effect: listening_ww
                else:
                  - light.turn_off: top_led

  - id: turn_on_notification
    then:
      - lambda: id(notification) = true;
      - script.execute: reset_led

  - id: clear_notification
    then:
      - lambda: id(notification) = false;
      - script.execute: reset_led

  - id: set_volume
    mode: restart
    parameters:
      volume: float
    then:
      - media_player.volume_set:
          id: onju_out
          volume: !lambda return clamp(id(onju_out).volume+volume, 0.0f, 1.0f);

  - id: show_volume
    mode: restart
    then:
      - light.turn_on:
          id: top_led
          effect: show_volume
      - delay: 1s
      - script.execute: reset_led

  - id: turn_on_wake_word
    then:
      - if:
          condition:
            and:
              - binary_sensor.is_off: mute_switch
              - switch.is_on: use_wake_word
          then:
            - micro_wake_word.start
            - script.execute: reset_led
          else:
            - logger.log:
                tag: "turn_on_wake_word"
                format: "Trying to start listening for wake word, but %s"
                args:
                  [
                    'id(mute_switch).state ? "mute switch is on" : "use wake word toggle is off"',
                  ]
                level: "INFO"

  - id: turn_off_wake_word
    then:
      - micro_wake_word.stop
      - script.execute: reset_led

  - id: calibrate_touch
    parameters:
      button: int
    then:
      - lambda: |-
          static uint8_t thresh_indices[3] = {0, 0, 0};
          static uint32_t sums[3] = {0, 0, 0};
          static uint8_t qsizes[3] = {0, 0, 0};
          static uint16_t consecutive_anomalies_per_button[3] = {0, 0, 0};

          uint32_t newval;
          uint32_t* calibration_values;
          switch(button) {
            case 0:
              newval = id(volume_down).get_value();
              calibration_values = id(touch_calibration_values_left);
              break;
            case 1:
              newval = id(action).get_value();
              calibration_values = id(touch_calibration_values_center);
              break;
            case 2:
              newval = id(volume_up).get_value();
              calibration_values = id(touch_calibration_values_right);
              break;
            default:
              ESP_LOGE("touch_calibration", "Invalid button ID (%d)", button);
              return;
          }

          if(newval == 0) return;

          //ESP_LOGD("touch_calibration", "[%d] qsize %d, sum %d, thresh_index %d, consecutive_anomalies %d", button, qsizes[button], sums[button], thresh_indices[button], consecutive_anomalies_per_button[button]);
          //ESP_LOGD("touch_calibration", "[%d] New value is %d", button, newval);

          if(qsizes[button] == 5) {
            float avg = float(sums[button])/float(qsizes[button]);
            if((fabs(float(newval)-avg)/avg) > id(thresh_percent)) {
              consecutive_anomalies_per_button[button]++;
              //ESP_LOGD("touch_calibration", "[%d] %d anomalies detected.", button, consecutive_anomalies_per_button[button]);
              if(consecutive_anomalies_per_button[button] < 10)
                return;
            } 
          }

          //ESP_LOGD("touch_calibration", "[%d] Resetting consecutive anomalies counter.", button);
          consecutive_anomalies_per_button[button] = 0;


          if(qsizes[button] == 5) {
            //ESP_LOGD("touch_calibration", "[%d] Queue full, removing %d.", button, id(touch_calibration_values)[thresh_indices[button]]);
            sums[button] -= (uint32_t) *(calibration_values+thresh_indices[button]);// id(touch_calibration_values)[thresh_indices[button]];
            qsizes[button]--;
          }
          *(calibration_values+thresh_indices[button]) = newval;
          sums[button] += newval;
          qsizes[button]++;
          thresh_indices[button] = (thresh_indices[button] + 1) % 5;

          //ESP_LOGD("touch_calibration", "[%d] Average value is %d", button, sums[button]/qsizes[button]);
          uint32_t newthresh = uint32_t((sums[button]/qsizes[button]) * (1.0 + id(thresh_percent)));
          //ESP_LOGD("touch_calibration", "[%d] Setting threshold %d", button, newthresh);

          switch(button) {
            case 0:
              id(volume_down).set_threshold(newthresh);
              break;
            case 1:
              id(action).set_threshold(newthresh);
              break;
            case 2:
              id(volume_up).set_threshold(newthresh);
              break;
            default:
              ESP_LOGE("touch_calibration", "Invalid button ID (%d)", button);
              return;
          }

switch:
  - platform: template
    name: Use Wake Word
    id: use_wake_word
    optimistic: true
    restore_mode: RESTORE_DEFAULT_ON
    on_turn_on:
      - script.execute: turn_on_wake_word
    on_turn_off:
      - script.execute: turn_off_wake_word
  - platform: gpio
    id: dac_mute
    restore_mode: ALWAYS_OFF
    pin:
      number: GPIO21
      inverted: True