From c275024cdad5c5cfef38e3c8ab5c11d969273345 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 7 Aug 2025 11:13:50 +1000 Subject: [PATCH 01/81] Create minimal example --- examples/minimal/CMakeLists.txt | 6 ++ examples/minimal/main/CMakeLists.txt | 2 + examples/minimal/main/board.c | 20 ++++ examples/minimal/main/board.h | 12 +++ examples/minimal/main/example.c | 85 ++++++++++++++++ examples/minimal/main/example.h | 13 +++ examples/minimal/main/idf_component.yml | 12 +++ examples/minimal/main/main.c | 31 ++++++ examples/minimal/main/media.c | 123 ++++++++++++++++++++++++ examples/minimal/main/media.h | 39 ++++++++ examples/minimal/partitions.csv | 5 + examples/minimal/sdkconfig.defaults | 30 ++++++ 12 files changed, 378 insertions(+) create mode 100755 examples/minimal/CMakeLists.txt create mode 100755 examples/minimal/main/CMakeLists.txt create mode 100755 examples/minimal/main/board.c create mode 100644 examples/minimal/main/board.h create mode 100644 examples/minimal/main/example.c create mode 100644 examples/minimal/main/example.h create mode 100644 examples/minimal/main/idf_component.yml create mode 100644 examples/minimal/main/main.c create mode 100644 examples/minimal/main/media.c create mode 100644 examples/minimal/main/media.h create mode 100755 examples/minimal/partitions.csv create mode 100644 examples/minimal/sdkconfig.defaults diff --git a/examples/minimal/CMakeLists.txt b/examples/minimal/CMakeLists.txt new file mode 100755 index 0000000..824598b --- /dev/null +++ b/examples/minimal/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) +set(COMPONENTS main) # Trim build +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(minimal) \ No newline at end of file diff --git a/examples/minimal/main/CMakeLists.txt b/examples/minimal/main/CMakeLists.txt new file mode 100755 index 0000000..6e75b98 --- /dev/null +++ b/examples/minimal/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRCS "main.c" "example.c" "board.c" "media.c" + INCLUDE_DIRS ".") \ No newline at end of file diff --git a/examples/minimal/main/board.c b/examples/minimal/main/board.c new file mode 100755 index 0000000..97b5301 --- /dev/null +++ b/examples/minimal/main/board.c @@ -0,0 +1,20 @@ +#include "esp_log.h" +#include "codec_init.h" +#include "codec_board.h" +#include "board.h" + +static const char *TAG = "board"; + +void board_init() +{ + ESP_LOGI(TAG, "Initializing board"); + + // Initialize codec board + set_codec_board_type(CONFIG_CODEC_BOARD_TYPE); + codec_init_cfg_t cfg = { + .in_mode = CODEC_I2S_MODE_TDM, + .in_use_tdm = true, + .reuse_dev = false + }; + init_codec(&cfg); +} \ No newline at end of file diff --git a/examples/minimal/main/board.h b/examples/minimal/main/board.h new file mode 100644 index 0000000..2830b42 --- /dev/null +++ b/examples/minimal/main/board.h @@ -0,0 +1,12 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// Initialize board. +void board_init(void); + +#ifdef __cplusplus +} +#endif diff --git a/examples/minimal/main/example.c b/examples/minimal/main/example.c new file mode 100644 index 0000000..e14eec7 --- /dev/null +++ b/examples/minimal/main/example.c @@ -0,0 +1,85 @@ +#include "esp_log.h" +#include "livekit.h" +#include "livekit_sandbox.h" +#include "media.h" +#include "board.h" +#include "example.h" + +static const char *TAG = "livekit_example"; + +static livekit_room_handle_t room_handle; + +/// Invoked when the room's connection state changes. +static void on_state_changed(livekit_connection_state_t state, void* ctx) +{ + ESP_LOGI(TAG, "Room state: %s", livekit_connection_state_str(state)); +} + +void join_room() +{ + if (room_handle != NULL) { + ESP_LOGE(TAG, "Room already created"); + return; + } + + livekit_room_options_t room_options = { + .publish = { + .kind = LIVEKIT_MEDIA_TYPE_AUDIO, + .audio_encode = { + .codec = LIVEKIT_AUDIO_CODEC_OPUS, + .sample_rate = 16000, + .channel_count = 1 + }, + .capturer = media_get_capturer() + }, + .subscribe = { + .kind = LIVEKIT_MEDIA_TYPE_AUDIO, + .renderer = media_get_renderer() + }, + .on_state_changed = on_state_changed + }; + if (livekit_room_create(&room_handle, &room_options) != LIVEKIT_ERR_NONE) { + ESP_LOGE(TAG, "Failed to create room"); + return; + } + + livekit_err_t connect_res; +#ifdef CONFIG_LK_USE_SANDBOX + // Option A: Sandbox token server. + livekit_sandbox_res_t res = {}; + livekit_sandbox_options_t gen_options = { + .sandbox_id = CONFIG_LK_SANDBOX_ID, + .room_name = CONFIG_LK_SANDBOX_ROOM_NAME, + .participant_name = CONFIG_LK_SANDBOX_PARTICIPANT_NAME + }; + if (!livekit_sandbox_generate(&gen_options, &res)) { + ESP_LOGE(TAG, "Failed to generate sandbox token"); + return; + } + connect_res = livekit_room_connect(room_handle, res.server_url, res.token); + livekit_sandbox_res_free(&res); +#else + // Option B: Pre-generated token. + connect_res = livekit_room_connect(room_handle, CONFIG_LK_SERVER_URL, CONFIG_LK_TOKEN); +#endif + + if (connect_res != LIVEKIT_ERR_NONE) { + ESP_LOGE(TAG, "Failed to connect to room"); + } +} + +void leave_room() +{ + if (room_handle == NULL) { + ESP_LOGE(TAG, "Room not created"); + return; + } + if (livekit_room_close(room_handle) != LIVEKIT_ERR_NONE) { + ESP_LOGE(TAG, "Failed to leave room"); + } + if (livekit_room_destroy(room_handle) != LIVEKIT_ERR_NONE) { + ESP_LOGE(TAG, "Failed to destroy room"); + return; + } + room_handle = NULL; +} \ No newline at end of file diff --git a/examples/minimal/main/example.h b/examples/minimal/main/example.h new file mode 100644 index 0000000..f576f95 --- /dev/null +++ b/examples/minimal/main/example.h @@ -0,0 +1,13 @@ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void join_room(); +void leave_room(); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/examples/minimal/main/idf_component.yml b/examples/minimal/main/idf_component.yml new file mode 100644 index 0000000..e5d5f8a --- /dev/null +++ b/examples/minimal/main/idf_component.yml @@ -0,0 +1,12 @@ +dependencies: + idf: ">=5.4" + livekit: + path: ../../../components/livekit + codec_board: + path: ../../../components/third_party/esp-webrtc-solution/components/codec_board + render_impl: + path: ../../../components/third_party/esp-webrtc-solution/components/av_render/render_impl + livekit_sandbox: + path: ../../../components/livekit_sandbox + common: + path: ../../common diff --git a/examples/minimal/main/main.c b/examples/minimal/main/main.c new file mode 100644 index 0000000..d19c58d --- /dev/null +++ b/examples/minimal/main/main.c @@ -0,0 +1,31 @@ +#include "esp_log.h" +#include "media_lib_adapter.h" +#include "media_lib_os.h" +#include "livekit.h" +#include "network.h" +#include "media.h" +#include "board.h" +#include "example.h" + +static void run_async_join_room(void *arg) +{ + join_room(); // See example.c + media_lib_thread_destroy(NULL); +} + +static int network_event_handler(bool connected) +{ + // Create and join the room once network is connected. + if (!connected) return 0; + media_lib_thread_create_from_scheduler(NULL, "join", run_async_join_room, NULL); + return 0; +} + +void app_main(void) +{ + esp_log_level_set("*", ESP_LOG_INFO); + livekit_system_init(); + board_init(); + media_init(); + network_init(CONFIG_WIFI_SSID, CONFIG_WIFI_PASSWORD, network_event_handler); +} diff --git a/examples/minimal/main/media.c b/examples/minimal/main/media.c new file mode 100644 index 0000000..b1e42f1 --- /dev/null +++ b/examples/minimal/main/media.c @@ -0,0 +1,123 @@ +#include "esp_check.h" +#include "esp_log.h" +#include "codec_init.h" +#include "esp_capture_path_simple.h" +#include "esp_capture_audio_enc.h" +#include "av_render_default.h" +#include "esp_audio_dec_default.h" +#include "esp_audio_enc_default.h" +#include "esp_capture_defaults.h" + +#include "media.h" + +static const char *TAG = "media"; + +#define NULL_CHECK(pointer, message) \ + ESP_RETURN_ON_FALSE(pointer != NULL, -1, TAG, message) + +typedef struct { + esp_capture_aenc_if_t *audio_encoder; + esp_capture_audio_src_if_t *audio_source; + esp_capture_path_if_t *capture_path; + esp_capture_path_handle_t capturer_handle; +} capture_system_t; + +typedef struct { + audio_render_handle_t audio_renderer; + av_render_handle_t av_renderer_handle; +} renderer_system_t; + +static capture_system_t capturer_system; +static renderer_system_t renderer_system; + +static int build_capturer_system(void) +{ + // 1. Create audio encoder + capturer_system.audio_encoder = esp_capture_new_audio_encoder(); + NULL_CHECK(capturer_system.audio_encoder, "Failed to create audio encoder"); + + // 2. Create audio source + esp_codec_dev_handle_t record_handle = get_record_handle(); + NULL_CHECK(record_handle, "Failed to get record handle"); + + esp_capture_audio_aec_src_cfg_t codec_cfg = { + .record_handle = record_handle, + .channel = 4, + .channel_mask = 1 | 2 + }; + capturer_system.audio_source = esp_capture_new_audio_aec_src(&codec_cfg); + NULL_CHECK(capturer_system.audio_source, "Failed to create audio source"); + + // 3. Create capture path + esp_capture_simple_path_cfg_t path_cfg = { + .aenc = capturer_system.audio_encoder, + }; + capturer_system.capture_path = esp_capture_build_simple_path(&path_cfg); + NULL_CHECK(capturer_system.capture_path, "Failed to create capture path"); + + // 4. Create capture system + esp_capture_cfg_t cfg = { + .sync_mode = ESP_CAPTURE_SYNC_MODE_AUDIO, + .audio_src = capturer_system.audio_source, + .capture_path = capturer_system.capture_path, + }; + esp_capture_open(&cfg, &capturer_system.capturer_handle); + NULL_CHECK(capturer_system.capturer_handle, "Failed to open capture system"); + return 0; +} + +static int build_renderer_system(void) +{ + // 1. Create audio renderer + i2s_render_cfg_t i2s_cfg = { + .play_handle = get_playback_handle() + }; + renderer_system.audio_renderer = av_render_alloc_i2s_render(&i2s_cfg); + NULL_CHECK(renderer_system.audio_renderer, "Failed to create I2S renderer"); + + // Set initial speaker volume + esp_codec_dev_set_out_vol(i2s_cfg.play_handle, CONFIG_DEFAULT_PLAYBACK_VOL); + + // 2. Create AV renderer + // For this example, this only includes an audio renderer. + av_render_cfg_t render_cfg = { + .audio_render = renderer_system.audio_renderer, + .audio_raw_fifo_size = 8 * 4096, + .audio_render_fifo_size = 100 * 1024, + .allow_drop_data = false, + }; + renderer_system.av_renderer_handle = av_render_open(&render_cfg); + NULL_CHECK(renderer_system.av_renderer_handle, "Failed to create AV renderer"); + + // 3. Set frame info + av_render_audio_frame_info_t frame_info = { + .sample_rate = 16000, + .channel = 2, + .bits_per_sample = 16, + }; + av_render_set_fixed_frame_info(renderer_system.av_renderer_handle, &frame_info); + + return 0; +} + +int media_init(void) +{ + // Register default audio encoder and decoder + esp_audio_enc_register_default(); + esp_audio_dec_register_default(); + + // Build capturer and renderer systems + build_capturer_system(); + build_renderer_system(); + return 0; +} + +esp_capture_handle_t media_get_capturer(void) +{ + return capturer_system.capturer_handle; +} + +av_render_handle_t media_get_renderer(void) +{ + return renderer_system.av_renderer_handle; +} \ No newline at end of file diff --git a/examples/minimal/main/media.h b/examples/minimal/main/media.h new file mode 100644 index 0000000..3b07788 --- /dev/null +++ b/examples/minimal/main/media.h @@ -0,0 +1,39 @@ + +#pragma once + +#include "esp_capture.h" +#include "av_render.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/// Initializes the capturer and renderer systems. +int media_init(void); + +/// Returns the capturer handle. +/// +/// This handle is provided to a LiveKit room when initialized to enable +/// publishing tracks from captured media (i.e. audio from a microphone and/or +/// video from a camera). +/// +/// How the capturer is configured is determined by the requirements of +/// your application and the hardware you are using. +/// +esp_capture_handle_t media_get_capturer(void); + +/// Returns the renderer handle. +/// +/// This handle is provided to a LiveKit room when initialized to enable +/// rendering media from subscribed tracks (i.e. playing audio through a +/// speaker and/or displaying video to a screen). +/// +/// How the renderer is configured is determined by the requirements of +/// your application and the hardware you are using. +/// +av_render_handle_t media_get_renderer(void); + +#ifdef __cplusplus +} +#endif + diff --git a/examples/minimal/partitions.csv b/examples/minimal/partitions.csv new file mode 100755 index 0000000..111c125 --- /dev/null +++ b/examples/minimal/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 3M, diff --git a/examples/minimal/sdkconfig.defaults b/examples/minimal/sdkconfig.defaults new file mode 100644 index 0000000..d9bb373 --- /dev/null +++ b/examples/minimal/sdkconfig.defaults @@ -0,0 +1,30 @@ +# This file was generated using idf.py save-defconfig. It can be edited manually. +# Espressif IoT Development Framework (ESP-IDF) 5.4.1 Project Minimal Configuration +# +CONFIG_IDF_TARGET="esp32s3" +CONFIG_CODEC_BOARD_TYPE="ESP32_S3_BOX_3" + +CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE=n +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_DEFAULT_PLAYBACK_VOL=100 +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESP_WS_CLIENT_ENABLE_DYNAMIC_BUFFER=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y +CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_FREERTOS_HZ=1000 +CONFIG_IDF_EXPERIMENTAL_FEATURES=y +CONFIG_LWIP_SNTP_MAX_SERVERS=2 +CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y +CONFIG_MBEDTLS_SSL_DTLS_SRTP=y +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y +CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=1024 +CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=8192 +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y +CONFIG_SPIRAM=y From 244b35503a181a4ac459013f9d67c8afe9a885c8 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 8 Aug 2025 11:26:19 +1000 Subject: [PATCH 02/81] Wip --- components/livekit/Kconfig | 6 + components/livekit/core/engine.c | 741 ++++++------------------------- components/livekit/core/engine.h | 8 + components/livekit/core/peer.h | 1 - 4 files changed, 153 insertions(+), 603 deletions(-) create mode 100644 components/livekit/Kconfig diff --git a/components/livekit/Kconfig b/components/livekit/Kconfig new file mode 100644 index 0000000..d9a2ded --- /dev/null +++ b/components/livekit/Kconfig @@ -0,0 +1,6 @@ +menu "LiveKit" + config LK_MAX_RETRIES + int "Maximum connection retries" + range 0 100 + default 7 +endmenu diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 54d513e..6aa7604 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1,632 +1,174 @@ +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include "freertos/event_groups.h" +#include +#include #include "esp_log.h" -#include "media_lib_os.h" -#include "esp_codec_dev.h" - -#include "protocol.h" +#include "url.h" #include "engine.h" -#include "signaling.h" -#include "peer.h" - -static const char *TAG = "livekit_engine"; -#define VIDEO_TRACK_CID "video0" -#define AUDIO_TRACK_CID "audio0" +// MARK: - Constants +static const char* TAG = "livekit_engine"; -#define VIDEO_TRACK_NAME "Video" -#define AUDIO_TRACK_NAME "Audio" -#define FRAME_INTERVAL_MS 20 +// MARK: - Event bits +#define ENGINE_EV_CONNECT_CMD (1 << 0) +#define ENGINE_EV_CLOSE_CMD (1 << 1) +#define ENGINE_EV_COMPONENT_STATE_CHANGED (1 << 2) -#define SAFE_FREE(ptr) if (ptr != NULL) { \ - free(ptr); \ - ptr = NULL; \ -} +// MARK: - Type definitions typedef struct { + engine_state_t state; engine_options_t options; - signal_handle_t sig; - - peer_handle_t pub_peer; - peer_handle_t sub_peer; - - esp_peer_ice_server_cfg_t *ice_servers; - int ice_server_count; - - esp_capture_path_handle_t capturer_path; - bool is_media_streaming; - esp_codec_dev_handle_t renderer_handle; - esp_peer_audio_stream_info_t sub_audio_info; + // signal_handle_t signal_handle; + // peer_handle_t pub_peer_handle; + // peer_handle_t sub_peer_handle; - connection_state_t state; /// Engine state, derived from signaling and peer states - connection_state_t sig_state; /// Signaling state - connection_state_t pub_state; /// Publisher peer state - connection_state_t sub_state; /// Subscriber peer state + SemaphoreHandle_t state_mutex; + char* signal_url; - bool is_subscriber_primary; - char local_participant_sid[32]; - char sub_audio_track_sid[32]; + TaskHandle_t task_handle; + EventGroupHandle_t event_group; + bool is_running; + int retry_count; } engine_t; -static void recalculate_state(engine_t *eng) -{ - connection_state_t new_state = eng->state; +// MARK: - State machine - if (eng->sig_state == CONNECTION_STATE_FAILED || - eng->pub_state == CONNECTION_STATE_FAILED || - eng->sub_state == CONNECTION_STATE_FAILED) - { - new_state = CONNECTION_STATE_FAILED; - } - else if (eng->sig_state == CONNECTION_STATE_RECONNECTING || - eng->pub_state == CONNECTION_STATE_RECONNECTING || - eng->sub_state == CONNECTION_STATE_RECONNECTING) - { - new_state = CONNECTION_STATE_RECONNECTING; - } +static void handle_state_disconnected(engine_t *eng); +static void handle_state_connecting(engine_t *eng); +static void handle_state_connected(engine_t *eng); +static void handle_state_reconnecting(engine_t *eng); +static void handle_state_disconnecting(engine_t *eng); - else if (eng->sig_state == CONNECTION_STATE_CONNECTED && - ( (eng->is_subscriber_primary && eng->sub_state == CONNECTION_STATE_CONNECTED) || - (!eng->is_subscriber_primary && eng->pub_state == CONNECTION_STATE_CONNECTED) )) - { - new_state = CONNECTION_STATE_CONNECTED; - } - else if (eng->sig_state == CONNECTION_STATE_DISCONNECTED && - eng->pub_state == CONNECTION_STATE_DISCONNECTED && - eng->sub_state == CONNECTION_STATE_DISCONNECTED) - { - new_state = CONNECTION_STATE_DISCONNECTED; - } - else { - new_state = CONNECTION_STATE_CONNECTING; - } - - if (new_state != eng->state) { - ESP_LOGI(TAG, "State changed: %d -> %d", eng->state, new_state); - eng->state = new_state; - eng->options.on_state_changed(eng->state, eng->options.ctx); - } -} - -static esp_capture_codec_type_t capture_audio_codec_type(esp_peer_audio_codec_t peer_codec) -{ - switch (peer_codec) { - case ESP_PEER_AUDIO_CODEC_G711A: return ESP_CAPTURE_CODEC_TYPE_G711A; - case ESP_PEER_AUDIO_CODEC_G711U: return ESP_CAPTURE_CODEC_TYPE_G711U; - case ESP_PEER_AUDIO_CODEC_OPUS: return ESP_CAPTURE_CODEC_TYPE_OPUS; - default: return ESP_CAPTURE_CODEC_TYPE_NONE; - } -} - -static esp_capture_codec_type_t capture_video_codec_type(esp_peer_video_codec_t peer_codec) -{ - switch (peer_codec) { - case ESP_PEER_VIDEO_CODEC_H264: return ESP_CAPTURE_CODEC_TYPE_H264; - case ESP_PEER_VIDEO_CODEC_MJPEG: return ESP_CAPTURE_CODEC_TYPE_MJPEG; - default: return ESP_CAPTURE_CODEC_TYPE_NONE; - } -} - -static av_render_audio_codec_t get_dec_codec(esp_peer_audio_codec_t codec) -{ - switch (codec) { - case ESP_PEER_AUDIO_CODEC_G711A: return AV_RENDER_AUDIO_CODEC_G711A; - case ESP_PEER_AUDIO_CODEC_G711U: return AV_RENDER_AUDIO_CODEC_G711U; - case ESP_PEER_AUDIO_CODEC_OPUS: return AV_RENDER_AUDIO_CODEC_OPUS; - default: return AV_RENDER_AUDIO_CODEC_NONE; - } -} - -static void convert_dec_aud_info(esp_peer_audio_stream_info_t *info, av_render_audio_info_t *dec_info) -{ - dec_info->codec = get_dec_codec(info->codec); - if (info->codec == ESP_PEER_AUDIO_CODEC_G711A || info->codec == ESP_PEER_AUDIO_CODEC_G711U) { - dec_info->sample_rate = 8000; - dec_info->channel = 1; - } else { - dec_info->sample_rate = info->sample_rate; - dec_info->channel = info->channel; - } - dec_info->bits_per_sample = 16; -} - -static void _media_stream_send_audio(engine_t *eng) -{ - esp_capture_stream_frame_t audio_frame = { - .stream_type = ESP_CAPTURE_STREAM_TYPE_AUDIO, - }; - while (esp_capture_acquire_path_frame(eng->capturer_path, &audio_frame, true) == ESP_CAPTURE_ERR_OK) { - esp_peer_audio_frame_t audio_send_frame = { - .pts = audio_frame.pts, - .data = audio_frame.data, - .size = audio_frame.size, - }; - peer_send_audio(eng->pub_peer, &audio_send_frame); - esp_capture_release_path_frame(eng->capturer_path, &audio_frame); - } -} - -static void _media_stream_send_video(engine_t *eng) -{ - esp_capture_stream_frame_t video_frame = { - .stream_type = ESP_CAPTURE_STREAM_TYPE_VIDEO, - }; - if (esp_capture_acquire_path_frame(eng->capturer_path, &video_frame, true) == ESP_CAPTURE_ERR_OK) { - esp_peer_video_frame_t video_send_frame = { - .pts = video_frame.pts, - .data = video_frame.data, - .size = video_frame.size, - }; - peer_send_video(eng->pub_peer, &video_send_frame); - esp_capture_release_path_frame(eng->capturer_path, &video_frame); - } -} - -static void media_stream_task(void *arg) +static void engine_task(void *arg) { engine_t *eng = (engine_t *)arg; - while (eng->is_media_streaming) { - if (eng->options.media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE) { - _media_stream_send_audio(eng); + while (eng->is_running) { + engine_state_t state = eng->state; + switch (state) { + case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng); break; + case ENGINE_STATE_CONNECTING: handle_state_connecting(eng); break; + case ENGINE_STATE_CONNECTED: handle_state_connected(eng); break; + case ENGINE_STATE_RECONNECTING: handle_state_reconnecting(eng); break; + case ENGINE_STATE_DISCONNECTING: handle_state_disconnecting(eng); break; + default: break; } - if (eng->options.media.video_info.codec != ESP_PEER_VIDEO_CODEC_NONE) { - _media_stream_send_video(eng); + if (eng->state != state) { + ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); + // TODO: Dispatch change event } - media_lib_thread_sleep(FRAME_INTERVAL_MS); } - media_lib_thread_destroy(NULL); + vTaskDelete(NULL); } -static engine_err_t media_stream_begin(engine_t *eng) +static void handle_state_disconnected(engine_t *eng) { - if (esp_capture_start(eng->options.media.capturer) != ESP_CAPTURE_ERR_OK) { - ESP_LOGE(TAG, "Failed to start capture"); - return ENGINE_ERR_MEDIA; + EventBits_t bits = xEventGroupWaitBits( + eng->event_group, + ENGINE_EV_CONNECT_CMD | ENGINE_EV_CLOSE_CMD, + pdTRUE, + pdFALSE, + portMAX_DELAY + ); + if (bits & ENGINE_EV_CONNECT_CMD) { + eng->state = ENGINE_STATE_CONNECTING; } - media_lib_thread_handle_t handle = NULL; - eng->is_media_streaming = true; - if (media_lib_thread_create_from_scheduler(&handle, STREAM_THREAD_NAME, media_stream_task, eng) != ESP_OK) { - ESP_LOGE(TAG, "Failed to create media stream thread"); - eng->is_media_streaming = false; - return ENGINE_ERR_MEDIA; - } - return ENGINE_ERR_NONE; } -static engine_err_t media_stream_end(engine_t *eng) +static void handle_state_connecting(engine_t *eng) { - if (!eng->is_media_streaming) { - return ENGINE_ERR_NONE; - } - eng->is_media_streaming = false; - esp_capture_stop(eng->options.media.capturer); - return ENGINE_ERR_NONE; -} + // TODO: Setup connections + // - Wait for required connections to be established + // - Handle new connection or close -static engine_err_t send_add_audio_track(engine_t *eng) -{ - bool is_stereo = eng->options.media.audio_info.channel == 2; - livekit_pb_add_track_request_t req = { - .cid = AUDIO_TRACK_CID, - .name = AUDIO_TRACK_NAME, - .type = LIVEKIT_PB_TRACK_TYPE_AUDIO, - .source = LIVEKIT_PB_TRACK_SOURCE_MICROPHONE, - .muted = false, - .audio_features_count = is_stereo ? 1 : 0, - .audio_features = { LIVEKIT_PB_AUDIO_TRACK_FEATURE_TF_STEREO }, - .layers_count = 0 - }; - - if (signal_send_add_track(eng->sig, &req) != SIGNAL_ERR_NONE) { - ESP_LOGE(TAG, "Failed to publish audio track"); - return ENGINE_ERR_SIGNALING; - } - return ENGINE_ERR_NONE; -} - -static engine_err_t send_add_video_track(engine_t *eng) -{ - livekit_pb_video_layer_t video_layer = { - .quality = LIVEKIT_PB_VIDEO_QUALITY_HIGH, - .width = eng->options.media.video_info.width, - .height = eng->options.media.video_info.height - }; - livekit_pb_add_track_request_t req = { - .cid = VIDEO_TRACK_CID, - .name = VIDEO_TRACK_NAME, - .type = LIVEKIT_PB_TRACK_TYPE_VIDEO, - .source = LIVEKIT_PB_TRACK_SOURCE_CAMERA, - .muted = false, - .layers_count = 1, - .layers = { video_layer }, - .audio_features_count = 0 - }; - - if (signal_send_add_track(eng->sig, &req) != SIGNAL_ERR_NONE) { - ESP_LOGE(TAG, "Failed to publish video track"); - return ENGINE_ERR_SIGNALING; - } - return ENGINE_ERR_NONE; -} - -/// Begins media streaming and sends add track requests. -static engine_err_t publish_tracks(engine_t *eng) -{ - if (eng->options.media.audio_info.codec == ESP_PEER_AUDIO_CODEC_NONE && - eng->options.media.video_info.codec == ESP_PEER_VIDEO_CODEC_NONE) { - ESP_LOGI(TAG, "No media tracks to publish"); - return ENGINE_ERR_NONE; - } - - int ret = ENGINE_ERR_OTHER; - do { - if (media_stream_begin(eng) != ENGINE_ERR_NONE) { - ret = ENGINE_ERR_MEDIA; - break; - } - if (eng->options.media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE && - send_add_audio_track(eng) != ENGINE_ERR_NONE) { - ret = ENGINE_ERR_SIGNALING; - break; - } - if (eng->options.media.video_info.codec != ESP_PEER_VIDEO_CODEC_NONE && - send_add_video_track(eng) != ENGINE_ERR_NONE) { - ret = ENGINE_ERR_SIGNALING; - break; - } - return ENGINE_ERR_NONE; - } while (0); - - media_stream_end(eng); - return ret; -} - -static engine_err_t subscribe_tracks(engine_t *eng, livekit_pb_track_info_t *tracks, int count) -{ - if (eng == NULL || tracks == NULL || count <= 0) { - return ENGINE_ERR_INVALID_ARG; - } - if (eng->sub_audio_track_sid[0] != '\0') { - return ENGINE_ERR_NONE; - } - for (int i = 0; i < count; i++) { - livekit_pb_track_info_t *track = &tracks[i]; - - if (track->type != LIVEKIT_PB_TRACK_TYPE_AUDIO) { - continue; - } - signal_send_update_subscription(eng->sig, track->sid, true); - strncpy(eng->sub_audio_track_sid, track->sid, sizeof(eng->sub_audio_track_sid)); - ESP_LOGI(TAG, "Subscribed to audio track"); - } - return ENGINE_ERR_NONE; + eng->state = ENGINE_STATE_CONNECTED; } -static void free_ice_servers(engine_t *eng) +static void handle_state_connected(engine_t *eng) { - if (eng == NULL || eng->ice_servers == NULL) { + EventBits_t bits = xEventGroupWaitBits( + eng->event_group, + ENGINE_EV_CONNECT_CMD | ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED, + pdTRUE, + pdFALSE, + portMAX_DELAY + ); + if (bits & ENGINE_EV_CLOSE_CMD) { + eng->state = ENGINE_STATE_DISCONNECTING; return; } - esp_peer_ice_server_cfg_t *ice_servers = eng->ice_servers; - for (int i = 0; i < eng->ice_server_count; i++) { - SAFE_FREE(ice_servers[i].stun_url); - SAFE_FREE(ice_servers[i].user); - SAFE_FREE(ice_servers[i].psw); - } - SAFE_FREE(eng->ice_servers); - eng->ice_server_count = 0; -} - -__attribute__((unused)) -static engine_err_t set_ice_servers(engine_t* eng, livekit_pb_ice_server_t *servers, int count) -{ - if (eng == NULL || servers == NULL || count <= 0) { - return ENGINE_ERR_INVALID_ARG; - } - // A single livekit_ice_server_t can contain multiple URLs, which - // will map to multiple esp_peer_ice_server_cfg_t entries. - size_t cfg_count = 0; - for (int i = 0; i < count; i++) { - if (servers[i].urls_count <= 0) { - return ENGINE_ERR_INVALID_ARG; - } - for (int j = 0; j < servers[i].urls_count; j++) { - if (servers[i].urls[j] == NULL) { - return ENGINE_ERR_INVALID_ARG; - } - cfg_count++; - } - } - - esp_peer_ice_server_cfg_t *cfgs = calloc(cfg_count, sizeof(esp_peer_ice_server_cfg_t)); - if (cfgs == NULL) { - return ENGINE_ERR_NO_MEM; - } - - int cfg_idx = 0; - for (int i = 0; i < count; i++) { - for (int j = 0; j < servers[i].urls_count; j++) { - bool has_auth = false; - cfgs[cfg_idx].stun_url = strdup(servers[i].urls[j]); - if (servers[i].username != NULL) { - cfgs[cfg_idx].user = strdup(servers[i].username); - has_auth = true; - } - if (servers[i].credential != NULL) { - cfgs[cfg_idx].psw = strdup(servers[i].credential); - has_auth = true; - } - ESP_LOGI(TAG, "Adding ICE server: has_auth=%d, url=%s", has_auth, servers[i].urls[j]); - cfg_idx++; - } - } - - free_ice_servers(eng); - eng->ice_servers = cfgs; - eng->ice_server_count = cfg_count; - - return ENGINE_ERR_NONE; -} - -static void on_peer_pub_state_changed(connection_state_t state, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - if (state == CONNECTION_STATE_CONNECTED) { - publish_tracks(eng); - } - eng->pub_state = state; - recalculate_state(eng); -} - -static void on_peer_sub_state_changed(connection_state_t state, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - eng->sub_state = state; - recalculate_state(eng); -} - -static void on_peer_pub_offer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - signal_send_offer(eng->sig, sdp); -} - -static void on_peer_sub_answer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - signal_send_answer(eng->sig, sdp); -} - -static void on_peer_ice_candidate(const char *candidate, void *ctx) -{ - ESP_LOGD(TAG, "Peer generated ice candidate: %s", candidate); -} - -static void on_peer_packet_received(livekit_pb_data_packet_t* packet, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - eng->options.on_data_packet(packet, eng->options.ctx); -} - -static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - - av_render_audio_info_t render_info = {}; - convert_dec_aud_info(info, &render_info); - ESP_LOGD(TAG, "Audio render info: codec=%d, sample_rate=%" PRIu32 ", channels=%" PRIu8, - render_info.codec, render_info.sample_rate, render_info.channel); - - if (av_render_add_audio_stream(eng->renderer_handle, &render_info) != ESP_MEDIA_ERR_OK) { - ESP_LOGE(TAG, "Failed to add audio stream to renderer"); + if (bits & ENGINE_EV_CONNECT_CMD) { + // TODO: Support new connection while already connected return; } - eng->sub_audio_info = *info; -} - -static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - if (eng->sub_audio_info.codec == ESP_PEER_AUDIO_CODEC_NONE) return; - // TODO: Check engine state before rendering - - av_render_audio_data_t audio_data = { - .pts = frame->pts, - .data = frame->data, - .size = frame->size, - }; - av_render_add_audio_data(eng->renderer_handle, &audio_data); -} - -static void on_sig_state_changed(connection_state_t state, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - eng->sig_state = state; - recalculate_state(eng); -} - -static bool disconnect_peer(peer_handle_t *peer) -{ - if (*peer == NULL) return false; - if (peer_disconnect(*peer) != PEER_ERR_NONE) return false; - if (peer_destroy(*peer) != PEER_ERR_NONE) return false; - *peer = NULL; - return true; -} - -static bool connect_peer(engine_t *eng, peer_options_t *options, peer_handle_t *peer) -{ - disconnect_peer(peer); - if (peer_create(peer, options) != PEER_ERR_NONE) return false; - if (peer_connect(*peer) != PEER_ERR_NONE) return false; - return true; + if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { + // TODO: Ensure components are still properly connected + } } -static void on_sig_join(livekit_pb_join_response_t *join_res, void *ctx) +static void handle_state_reconnecting(engine_t *eng) { - engine_t *eng = (engine_t *)ctx; - - if (join_res->subscriber_primary) { - eng->is_subscriber_primary = true; - ESP_LOGE(TAG, "Subscriber primary is not supported yet"); - return; - } - - // set_ice_servers(eng, join_res->ice_servers, join_res->ice_servers_count); - - peer_options_t options = { - // Options common to both peers - .force_relay = join_res->has_client_configuration && - join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED, - .media = &eng->options.media, - .server_list = eng->ice_servers, - .server_count = eng->ice_server_count, - .on_ice_candidate = on_peer_ice_candidate, - .on_packet_received = on_peer_packet_received, - .ctx = eng - }; - - // 1. Publisher peer - options.is_primary = !join_res->subscriber_primary; - options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; - options.on_state_changed = on_peer_pub_state_changed; - options.on_sdp = on_peer_pub_offer; - - if (!connect_peer(eng, &options, &eng->pub_peer)) { - ESP_LOGE(TAG, "Failed to connect publisher peer"); - return; - } - - // 2. Subscriber peer - options.is_primary = join_res->subscriber_primary; - options.target = LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER; - options.on_state_changed = on_peer_sub_state_changed; - options.on_sdp = on_peer_sub_answer; - options.on_audio_info = on_peer_sub_audio_info; - options.on_audio_frame = on_peer_sub_audio_frame; - - if (!connect_peer(eng, &options, &eng->sub_peer)) { - ESP_LOGE(TAG, "Failed to connect subscriber peer"); + if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { + ESP_LOGW(TAG, "Max retries reached"); + eng->state = ENGINE_STATE_DISCONNECTED; return; } - // Store fields from join response required for later use - strncpy(eng->local_participant_sid, join_res->participant.sid, sizeof(eng->local_participant_sid)); - - // Join response contains initial room and participant info (both local and remote); - // extract and invoke the appropriate handlers. - eng->options.on_room_info(&join_res->room, eng->options.ctx); - eng->options.on_participant_info(&join_res->participant, true, eng->options.ctx); - for (int i = 0; i < join_res->other_participants_count; i++) { - eng->options.on_participant_info(&join_res->other_participants[i], false, eng->options.ctx); - } -} - -static void on_sig_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_leave_request_action_t action, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - // TODO: Handle reconnect, update engine state - disconnect_peer(&eng->pub_peer); - disconnect_peer(&eng->sub_peer); - - memset(eng->local_participant_sid, 0, sizeof(eng->local_participant_sid)); - memset(eng->sub_audio_track_sid, 0, sizeof(eng->sub_audio_track_sid)); -} - -static void on_sig_room_update(const livekit_pb_room_t* info, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - eng->options.on_room_info(info, eng->options.ctx); -} + // TODO: Exponential backoff + uint32_t backoff_ms = 1000; -static void on_sig_participant_update(const livekit_pb_participant_info_t* info, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - bool is_local = strncmp(info->sid, eng->local_participant_sid, sizeof(eng->local_participant_sid)) == 0; - eng->options.on_participant_info(info, is_local, eng->options.ctx); + ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu32 "ms", + eng->retry_count + 1, CONFIG_LK_MAX_RETRIES, backoff_ms); - if (is_local) return; - subscribe_tracks(eng, info->tracks, info->tracks_count); -} + vTaskDelay(pdMS_TO_TICKS(backoff_ms)); + eng->retry_count++; -static void on_sig_answer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - peer_handle_sdp(eng->pub_peer, sdp); + // TODO: Try connection, transition to connected if successful + // - Handle full vs partial reconnect } -static void on_sig_offer(const char *sdp, void *ctx) +static void handle_state_disconnecting(engine_t *eng) { - engine_t *eng = (engine_t *)ctx; - peer_handle_sdp(eng->sub_peer, sdp); + // TODO: Graceful shutdown + // - Send leave, wait for ack + // - Disconnect components, wait for close with timeout } -static void on_sig_trickle(const char *ice_candidate, livekit_pb_signal_target_t target, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - peer_handle_t target_peer = target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? - eng->sub_peer : eng->pub_peer; - peer_handle_ice_candidate(target_peer, ice_candidate); -} +// MARK: - Public API engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) { - if (handle == NULL || - options == NULL || - options->on_state_changed == NULL || - options->on_data_packet == NULL || - options->on_room_info == NULL || - options->on_participant_info == NULL - ) { - return ENGINE_ERR_INVALID_ARG; - } - engine_t *eng = (engine_t *)calloc(1, sizeof(engine_t)); if (eng == NULL) { return ENGINE_ERR_NO_MEM; } - eng->options = *options; - signal_options_t sig_options = { - .ctx = eng, - .on_state_changed = on_sig_state_changed, - .on_join = on_sig_join, - .on_leave = on_sig_leave, - .on_room_update = on_sig_room_update, - .on_participant_update = on_sig_participant_update, - .on_answer = on_sig_answer, - .on_offer = on_sig_offer, - .on_trickle = on_sig_trickle - }; - - if (signal_create(&eng->sig, &sig_options) != SIGNAL_ERR_NONE) { - ESP_LOGE(TAG, "Failed to create signaling client"); + eng->state_mutex = xSemaphoreCreateMutex(); + if (eng->state_mutex == NULL) { + free(eng); + return ENGINE_ERR_NO_MEM; + } + eng->event_group = xEventGroupCreate(); + if (eng->event_group == NULL) { + vSemaphoreDelete(eng->state_mutex); free(eng); - return ENGINE_ERR_SIGNALING; + return ENGINE_ERR_NO_MEM; } - esp_capture_sink_cfg_t sink_cfg = { - .audio_info = { - .codec = capture_audio_codec_type(eng->options.media.audio_info.codec), - .sample_rate = eng->options.media.audio_info.sample_rate, - .channel = eng->options.media.audio_info.channel, - .bits_per_sample = 16, - }, - .video_info = { - .codec = capture_video_codec_type(eng->options.media.video_info.codec), - .width = eng->options.media.video_info.width, - .height = eng->options.media.video_info.height, - .fps = eng->options.media.video_info.fps, - }, - }; - - if (options->media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE) { - // TODO: Can we ensure the renderer is valid? If not, return error. - eng->renderer_handle = options->media.renderer; + eng->options = *options; + eng->state = ENGINE_STATE_DISCONNECTED; + eng->is_running = true; + + if (xTaskCreate(engine_task, "engine_task", 4096, eng, 5, &eng->task_handle) != pdPASS) { + vEventGroupDelete(eng->event_group); + vSemaphoreDelete(eng->state_mutex); + free(eng); + return ENGINE_ERR_NO_MEM; } - esp_capture_setup_path(eng->options.media.capturer, ESP_CAPTURE_PATH_PRIMARY, &sink_cfg, &eng->capturer_path); - esp_capture_enable_path(eng->capturer_path, ESP_CAPTURE_RUN_TYPE_ALWAYS); - // TODO: Handle capturer error - *handle = eng; + *handle = (engine_handle_t)eng; return ENGINE_ERR_NONE; } @@ -637,32 +179,46 @@ engine_err_t engine_destroy(engine_handle_t handle) } engine_t *eng = (engine_t *)handle; - if (eng->pub_peer != NULL) { - peer_destroy(eng->pub_peer); + eng->is_running = false; + if (eng->task_handle != NULL) { + // TODO: Wait for disconnected state or timeout + vTaskDelay(pdMS_TO_TICKS(100)); } - if (eng->sub_peer != NULL) { - peer_destroy(eng->sub_peer); + + xSemaphoreTake(eng->state_mutex, portMAX_DELAY); + if (eng->signal_url != NULL) { + free(eng->signal_url); } - signal_destroy(eng->sig); - free_ice_servers(eng); + vSemaphoreDelete(eng->state_mutex); + + // TODO: Free other resources free(eng); return ENGINE_ERR_NONE; } engine_err_t engine_connect(engine_handle_t handle, const char* server_url, const char* token) { - if (handle == NULL || server_url == NULL || token == NULL) { + if (handle == NULL) { return ENGINE_ERR_INVALID_ARG; } engine_t *eng = (engine_t *)handle; - if (eng->state != CONNECTION_STATE_DISCONNECTED) { + char* new_signal_url = NULL; + if (!url_build(server_url, token, &new_signal_url)) { + return ENGINE_ERR_INVALID_ARG; + } + + if (xSemaphoreTake(eng->state_mutex, pdMS_TO_TICKS(100)) != pdPASS) { + free(new_signal_url); return ENGINE_ERR_OTHER; } - if (signal_connect(eng->sig, server_url, token) != SIGNAL_ERR_NONE) { - ESP_LOGE(TAG, "Failed to connect signaling client"); - return ENGINE_ERR_SIGNALING; + if (eng->signal_url != NULL) { + free(eng->signal_url); } + eng->signal_url = new_signal_url; + xSemaphoreGive(eng->state_mutex); + + xEventGroupSetBits(eng->event_group, ENGINE_EV_CONNECT_CMD); return ENGINE_ERR_NONE; } @@ -673,37 +229,18 @@ engine_err_t engine_close(engine_handle_t handle) } engine_t *eng = (engine_t *)handle; - if (eng->state == CONNECTION_STATE_DISCONNECTED) { - return ENGINE_ERR_OTHER; - } - media_stream_end(eng); - // TODO: Reset just the stream that was added in case users added their own streams? - av_render_reset(eng->renderer_handle); - - if (eng->sub_peer != NULL) { - peer_disconnect(eng->sub_peer); - } - if (eng->pub_peer != NULL) { - peer_disconnect(eng->pub_peer); - } - if (eng->sig != NULL) { - // TODO: Ensure the WebSocket stays open long enough for the leave message to be sent - signal_send_leave(eng->sig); - signal_close(eng->sig); - } + xEventGroupSetBits(eng->event_group, ENGINE_EV_CLOSE_CMD); return ENGINE_ERR_NONE; } engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind) { - if (handle == NULL || packet == NULL) { + if (handle == NULL) { return ENGINE_ERR_INVALID_ARG; } engine_t *eng = (engine_t *)handle; - if (eng->pub_peer == NULL || - peer_send_data_packet(eng->pub_peer, packet, kind) != PEER_ERR_NONE) { - return ENGINE_ERR_RTC; - } + // TODO: Send data packet + return ENGINE_ERR_NONE; } \ No newline at end of file diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 5387a64..d3d1c72 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -18,6 +18,14 @@ extern "C" { /// Handle to an engine instance. typedef void *engine_handle_t; +typedef enum { + ENGINE_STATE_DISCONNECTED, + ENGINE_STATE_CONNECTING, + ENGINE_STATE_CONNECTED, + ENGINE_STATE_RECONNECTING, + ENGINE_STATE_DISCONNECTING +} engine_state_t; + typedef enum { ENGINE_ERR_NONE = 0, ENGINE_ERR_INVALID_ARG = -1, diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index e8503d6..b3e83b9 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -2,7 +2,6 @@ #pragma once #include "common.h" -#include "engine.h" #include "protocol.h" #define PEER_THREAD_NAME_PREFIX "lk_peer_" From 40eab0a82453d3e8c3e4bff63db4224f209e42c5 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 8 Aug 2025 14:24:43 +1000 Subject: [PATCH 03/81] Connect signal --- components/livekit/core/engine.c | 187 ++++++++++++++++++++++++++----- 1 file changed, 159 insertions(+), 28 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 6aa7604..aa24ecb 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -6,6 +6,7 @@ #include #include "esp_log.h" #include "url.h" +#include "signaling.h" #include "engine.h" // MARK: - Constants @@ -15,6 +16,8 @@ static const char* TAG = "livekit_engine"; #define ENGINE_EV_CONNECT_CMD (1 << 0) #define ENGINE_EV_CLOSE_CMD (1 << 1) #define ENGINE_EV_COMPONENT_STATE_CHANGED (1 << 2) +#define ENGINE_EV_JOIN_RECEIVED (1 << 3) +#define ENGINE_EV_LEAVE_RECEIVED (1 << 4) // MARK: - Type definitions @@ -22,12 +25,20 @@ typedef struct { engine_state_t state; engine_options_t options; - // signal_handle_t signal_handle; + signal_handle_t signal_handle; // peer_handle_t pub_peer_handle; // peer_handle_t sub_peer_handle; + connection_state_t signal_state; + + // Session state + livekit_pb_disconnect_reason_t disconnect_reason; + livekit_pb_leave_request_action_t leave_action; + bool is_subscriber_primary; + SemaphoreHandle_t state_mutex; - char* signal_url; + char* server_url; + char* token; TaskHandle_t task_handle; EventGroupHandle_t event_group; @@ -35,6 +46,31 @@ typedef struct { int retry_count; } engine_t; +// MARK: - Signal event handlers + +static void on_signal_state_changed(connection_state_t state, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + eng->signal_state = state; + xEventGroupSetBits(eng->event_group, ENGINE_EV_COMPONENT_STATE_CHANGED); +} + +static void on_signal_join(livekit_pb_join_response_t *join_res, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + eng->is_subscriber_primary = join_res->subscriber_primary; + // TODO: Retain other fields + xEventGroupSetBits(eng->event_group, ENGINE_EV_JOIN_RECEIVED); +} + +static void on_signal_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_leave_request_action_t action, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + eng->disconnect_reason = reason; + eng->leave_action = action; + xEventGroupSetBits(eng->event_group, ENGINE_EV_LEAVE_RECEIVED); +} + // MARK: - State machine static void handle_state_disconnected(engine_t *eng); @@ -59,7 +95,9 @@ static void engine_task(void *arg) if (eng->state != state) { ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); // TODO: Dispatch change event + continue; } + ESP_LOGI(TAG, "Re-entering state %d", eng->state); } vTaskDelete(NULL); } @@ -80,18 +118,80 @@ static void handle_state_disconnected(engine_t *eng) static void handle_state_connecting(engine_t *eng) { - // TODO: Setup connections - // - Wait for required connections to be established - // - Handle new connection or close + if (xSemaphoreTake(eng->state_mutex, pdMS_TO_TICKS(100)) != pdPASS) { + eng->state = ENGINE_STATE_DISCONNECTED; + return; + } + if (signal_connect(eng->signal_handle, eng->server_url, eng->token) != SIGNAL_ERR_NONE) { + ESP_LOGE(TAG, "Failed to connect signal client"); + eng->state = ENGINE_STATE_DISCONNECTED; + xSemaphoreGive(eng->state_mutex); + return; + } + xSemaphoreGive(eng->state_mutex); + + // 1. Wait for signal connected + while (1) { + EventBits_t bits = xEventGroupWaitBits( + eng->event_group, + ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED, + pdTRUE, + pdFALSE, + portMAX_DELAY + ); + + if (bits & ENGINE_EV_CLOSE_CMD) { + eng->state = ENGINE_STATE_DISCONNECTING; + return; + } + if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { + if (eng->signal_state == CONNECTION_STATE_CONNECTED) { + break; + } + if (eng->signal_state == CONNECTION_STATE_FAILED || + eng->signal_state == CONNECTION_STATE_DISCONNECTED) { + // TODO: Check error code (4xx is user error and should go to disconnecting) + eng->state = ENGINE_STATE_RECONNECTING; + return; + } + } + } + + // 2. Wait for join response + while (1) { + EventBits_t bits = xEventGroupWaitBits( + eng->event_group, + ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_JOIN_RECEIVED, + pdTRUE, + pdFALSE, + portMAX_DELAY + ); + if (bits & ENGINE_EV_CLOSE_CMD) { + eng->state = ENGINE_STATE_DISCONNECTING; + return; + } + if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { + if (eng->signal_state != CONNECTION_STATE_CONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + return; + } + } + if (bits & ENGINE_EV_JOIN_RECEIVED) { + break; + } + } + // TODO: Connect pub/sub (just go directly to connected for now) eng->state = ENGINE_STATE_CONNECTED; } static void handle_state_connected(engine_t *eng) { + // TODO: Track pub/sub + EventBits_t bits = xEventGroupWaitBits( eng->event_group, - ENGINE_EV_CONNECT_CMD | ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED, + ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_LEAVE_RECEIVED, pdTRUE, pdFALSE, portMAX_DELAY @@ -100,12 +200,23 @@ static void handle_state_connected(engine_t *eng) eng->state = ENGINE_STATE_DISCONNECTING; return; } - if (bits & ENGINE_EV_CONNECT_CMD) { - // TODO: Support new connection while already connected + if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { + if (eng->signal_state != CONNECTION_STATE_CONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + } return; } - if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { - // TODO: Ensure components are still properly connected + if (bits & ENGINE_EV_LEAVE_RECEIVED) { + switch (eng->leave_action) { + case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RECONNECT: + case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RESUME: + eng->state = ENGINE_STATE_RECONNECTING; + break; + default: + eng->state = ENGINE_STATE_DISCONNECTING; + break; + } + return; } } @@ -132,9 +243,13 @@ static void handle_state_reconnecting(engine_t *eng) static void handle_state_disconnecting(engine_t *eng) { - // TODO: Graceful shutdown - // - Send leave, wait for ack - // - Disconnect components, wait for close with timeout + // TODO: Send leave if user initiated + + if (eng->signal_state != CONNECTION_STATE_DISCONNECTED) { + signal_close(eng->signal_handle); + // TODO: Wait until disconnected + } + eng->state = ENGINE_STATE_DISCONNECTED; } // MARK: - Public API @@ -157,6 +272,20 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) return ENGINE_ERR_NO_MEM; } + signal_options_t signal_options = { + .ctx = eng, + .on_state_changed = on_signal_state_changed, + .on_join = on_signal_join, + .on_leave = on_signal_leave, + // TODO: Add other handlers + }; + if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { + vEventGroupDelete(eng->event_group); + vSemaphoreDelete(eng->state_mutex); + free(eng); + return ENGINE_ERR_SIGNALING; + } + eng->options = *options; eng->state = ENGINE_STATE_DISCONNECTED; eng->is_running = true; @@ -185,12 +314,17 @@ engine_err_t engine_destroy(engine_handle_t handle) vTaskDelay(pdMS_TO_TICKS(100)); } + signal_destroy(eng->signal_handle); + xSemaphoreTake(eng->state_mutex, portMAX_DELAY); - if (eng->signal_url != NULL) { - free(eng->signal_url); + if (eng->server_url != NULL) { + free(eng->server_url); + } + if (eng->token != NULL) { + free(eng->token); } vSemaphoreDelete(eng->state_mutex); - + vEventGroupDelete(eng->event_group); // TODO: Free other resources free(eng); return ENGINE_ERR_NONE; @@ -198,24 +332,22 @@ engine_err_t engine_destroy(engine_handle_t handle) engine_err_t engine_connect(engine_handle_t handle, const char* server_url, const char* token) { - if (handle == NULL) { + if (handle == NULL || server_url == NULL || token == NULL) { return ENGINE_ERR_INVALID_ARG; } engine_t *eng = (engine_t *)handle; - char* new_signal_url = NULL; - if (!url_build(server_url, token, &new_signal_url)) { - return ENGINE_ERR_INVALID_ARG; - } - if (xSemaphoreTake(eng->state_mutex, pdMS_TO_TICKS(100)) != pdPASS) { - free(new_signal_url); return ENGINE_ERR_OTHER; } - if (eng->signal_url != NULL) { - free(eng->signal_url); + if (eng->server_url != NULL) { + free(eng->server_url); } - eng->signal_url = new_signal_url; + if (eng->token != NULL) { + free(eng->token); + } + eng->server_url = strdup(server_url); + eng->token = strdup(token); xSemaphoreGive(eng->state_mutex); xEventGroupSetBits(eng->event_group, ENGINE_EV_CONNECT_CMD); @@ -238,8 +370,7 @@ engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_da if (handle == NULL) { return ENGINE_ERR_INVALID_ARG; } - engine_t *eng = (engine_t *)handle; - + // engine_t *eng = (engine_t *)handle; // TODO: Send data packet return ENGINE_ERR_NONE; From e3e741ae00c2cc9510e574ce63936a96564489e1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 8 Aug 2025 14:24:52 +1000 Subject: [PATCH 04/81] Signal refactor --- components/livekit/core/signaling.c | 36 +++++++++++++++++------------ components/livekit/core/signaling.h | 1 + 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index fb48a8b..8d38885 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -107,21 +107,28 @@ static void handle_res(signal_t *sg, livekit_pb_signal_response_t *res) sg->ping_timeout_ms = join_res->ping_timeout * 1000; esp_timer_start_periodic(sg->ping_timer, sg->ping_interval_ms * 1000); - sg->options.on_join(join_res, sg->options.ctx); + if (sg->options.on_join != NULL) { + sg->options.on_join(join_res, sg->options.ctx); + } break; case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: livekit_pb_leave_request_t *leave_res = &res->message.leave; ESP_LOGI(TAG, "Leave: reason=%d, action=%d", leave_res->reason, leave_res->action); esp_timer_stop(sg->ping_timer); - sg->options.on_leave(leave_res->reason, leave_res->action, sg->options.ctx); + if (sg->options.on_leave != NULL) { + sg->options.on_leave(leave_res->reason, leave_res->action, sg->options.ctx); + } break; case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: livekit_pb_room_update_t *room_update = &res->message.room_update; if (!room_update->has_room) break; - sg->options.on_room_update(&room_update->room, sg->options.ctx); + if (sg->options.on_room_update != NULL) { + sg->options.on_room_update(&room_update->room, sg->options.ctx); + } break; case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: livekit_pb_participant_update_t *participant_update = &res->message.update; + if (sg->options.on_participant_update == NULL) break; for (int i = 0; i < participant_update->participants_count; i++) { sg->options.on_participant_update(&participant_update->participants[i], sg->options.ctx); } @@ -129,12 +136,16 @@ static void handle_res(signal_t *sg, livekit_pb_signal_response_t *res) case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: livekit_pb_session_description_t *offer = &res->message.offer; ESP_LOGI(TAG, "Offer: id=%" PRIu32 "\n%s", offer->id, offer->sdp); - sg->options.on_offer(offer->sdp, sg->options.ctx); + if (sg->options.on_offer != NULL) { + sg->options.on_offer(offer->sdp, sg->options.ctx); + } break; case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: livekit_pb_session_description_t *answer = &res->message.answer; ESP_LOGI(TAG, "Answer: id=%" PRIu32 "\n%s", answer->id, answer->sdp); - sg->options.on_answer(answer->sdp, sg->options.ctx); + if (sg->options.on_answer != NULL) { + sg->options.on_answer(answer->sdp, sg->options.ctx); + } break; case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: livekit_pb_trickle_request_t *trickle = &res->message.trickle; @@ -162,7 +173,9 @@ static void handle_res(signal_t *sg, livekit_pb_signal_response_t *res) trickle->final, candidate->valuestring ); - sg->options.on_trickle(candidate->valuestring, trickle->target, sg->options.ctx); + if (sg->options.on_trickle != NULL) { + sg->options.on_trickle(candidate->valuestring, trickle->target, sg->options.ctx); + } } while (0); cJSON_Delete(candidate_init); break; @@ -240,15 +253,7 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) return SIGNAL_ERR_INVALID_ARG; } - if (options->on_state_changed == NULL || - options->on_join == NULL || - options->on_leave == NULL || - options->on_room_update == NULL || - options->on_participant_update == NULL || - options->on_offer == NULL || - options->on_answer == NULL || - options->on_trickle == NULL - ) { + if (options->on_state_changed == NULL) { ESP_LOGE(TAG, "Missing required event handlers"); return SIGNAL_ERR_INVALID_ARG; } @@ -276,6 +281,7 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) .disable_pingpong_discon = true, .reconnect_timeout_ms = SIGNAL_WS_RECONNECT_TIMEOUT_MS, .network_timeout_ms = SIGNAL_WS_NETWORK_TIMEOUT_MS, + .disable_auto_reconnect = true, #ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE .crt_bundle_attach = esp_crt_bundle_attach #endif diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index f528e34..b977d47 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -1,6 +1,7 @@ #pragma once +#include "protocol.h" #include "common.h" #ifdef __cplusplus From 16df9d3e4c359a202bfd2638c310d91166004f15 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 8 Aug 2025 16:16:54 +1000 Subject: [PATCH 05/81] Connect primary peer --- components/livekit/core/engine.c | 124 ++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 3 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index aa24ecb..23c3890 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -7,6 +7,7 @@ #include "esp_log.h" #include "url.h" #include "signaling.h" +#include "peer.h" #include "engine.h" // MARK: - Constants @@ -26,15 +27,17 @@ typedef struct { engine_options_t options; signal_handle_t signal_handle; - // peer_handle_t pub_peer_handle; + peer_handle_t pub_peer_handle; // peer_handle_t sub_peer_handle; connection_state_t signal_state; + connection_state_t pub_peer_state; // Session state livekit_pb_disconnect_reason_t disconnect_reason; livekit_pb_leave_request_action_t leave_action; bool is_subscriber_primary; + bool force_relay; SemaphoreHandle_t state_mutex; char* server_url; @@ -59,6 +62,8 @@ static void on_signal_join(livekit_pb_join_response_t *join_res, void *ctx) { engine_t *eng = (engine_t *)ctx; eng->is_subscriber_primary = join_res->subscriber_primary; + eng->force_relay = join_res->has_client_configuration && + join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; // TODO: Retain other fields xEventGroupSetBits(eng->event_group, ENGINE_EV_JOIN_RECEIVED); } @@ -71,6 +76,78 @@ static void on_signal_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_le xEventGroupSetBits(eng->event_group, ENGINE_EV_LEAVE_RECEIVED); } +// MARK: - Peer event handlers + +static void on_peer_ice_candidate(const char *candidate, void *ctx) +{ + ESP_LOGI(TAG, "ICE candidate"); + // TODO: Handle ICE candidate +} + +static void on_peer_packet_received(livekit_pb_data_packet_t* packet, void *ctx) +{ + ESP_LOGI(TAG, "Packet received"); + // TODO: Handle packet +} + +static void on_peer_pub_state_changed(connection_state_t state, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + eng->pub_peer_state = state; + xEventGroupSetBits(eng->event_group, ENGINE_EV_COMPONENT_STATE_CHANGED); +} + +static void on_peer_pub_offer(const char *sdp, void *ctx) +{ + ESP_LOGI(TAG, "Publisher peer offer"); + // TODO: Handle offer +} + +// MARK: - Peer lifecycle + +static bool disconnect_peer(peer_handle_t *peer) +{ + if (*peer == NULL) return false; + if (peer_disconnect(*peer) != PEER_ERR_NONE) return false; + if (peer_destroy(*peer) != PEER_ERR_NONE) return false; + *peer = NULL; + return true; +} + +static bool connect_peer(engine_t *eng, peer_options_t *options, peer_handle_t *peer) +{ + disconnect_peer(peer); + if (peer_create(peer, options) != PEER_ERR_NONE) return false; + if (peer_connect(*peer) != PEER_ERR_NONE) return false; + return true; +} + +static bool connect_peers(engine_t *eng) +{ + peer_options_t options = { + // Options common to both peers + .force_relay = eng->force_relay, + .media = &eng->options.media, + // .server_list = eng->ice_servers, + //.server_count = eng->ice_server_count, + .on_ice_candidate = on_peer_ice_candidate, + .on_packet_received = on_peer_packet_received, + .ctx = eng + }; + + // 1. Publisher peer + options.is_primary = !eng->is_subscriber_primary; + options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; + options.on_state_changed = on_peer_pub_state_changed; + options.on_sdp = on_peer_pub_offer; + + if (!connect_peer(eng, &options, &eng->pub_peer_handle)) { + ESP_LOGE(TAG, "Failed to connect publisher peer"); + return false; + } + return true; +} + // MARK: - State machine static void handle_state_disconnected(engine_t *eng); @@ -181,7 +258,46 @@ static void handle_state_connecting(engine_t *eng) } } - // TODO: Connect pub/sub (just go directly to connected for now) + // 3. Start peer connections, wait for primary connected + if (!connect_peers(eng)) { + eng->state = ENGINE_STATE_DISCONNECTING; + return; + } + while (1) { + EventBits_t bits = xEventGroupWaitBits( + eng->event_group, + ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_LEAVE_RECEIVED, + pdTRUE, + pdFALSE, + portMAX_DELAY + ); + if (bits & ENGINE_EV_CLOSE_CMD) { + eng->state = ENGINE_STATE_DISCONNECTING; + return; + } + if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { + if (eng->signal_state != CONNECTION_STATE_CONNECTED || + eng->pub_peer_state == CONNECTION_STATE_FAILED) { + eng->state = ENGINE_STATE_RECONNECTING; + return; + } + if (eng->pub_peer_state == CONNECTION_STATE_CONNECTED) { + break; + } + } + if (bits & ENGINE_EV_LEAVE_RECEIVED) { + // TODO: Factor out this common code + switch (eng->leave_action) { + case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RECONNECT: + case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RESUME: + eng->state = ENGINE_STATE_RECONNECTING; + break; + default: + eng->state = ENGINE_STATE_DISCONNECTING; + } + return; + } + } eng->state = ENGINE_STATE_CONNECTED; } @@ -214,7 +330,6 @@ static void handle_state_connected(engine_t *eng) break; default: eng->state = ENGINE_STATE_DISCONNECTING; - break; } return; } @@ -245,6 +360,8 @@ static void handle_state_disconnecting(engine_t *eng) { // TODO: Send leave if user initiated + disconnect_peer(&eng->pub_peer_handle); + if (eng->signal_state != CONNECTION_STATE_DISCONNECTED) { signal_close(eng->signal_handle); // TODO: Wait until disconnected @@ -315,6 +432,7 @@ engine_err_t engine_destroy(engine_handle_t handle) } signal_destroy(eng->signal_handle); + peer_destroy(eng->pub_peer_handle); xSemaphoreTake(eng->state_mutex, portMAX_DELAY); if (eng->server_url != NULL) { From c7da5a4a067728369e899352cfe3c7ff7a35373c Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:22:27 +1000 Subject: [PATCH 06/81] Remove dead code --- components/livekit/core/peer.c | 1 - components/livekit/core/peer.h | 4 ---- 2 files changed, 5 deletions(-) diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index f823813..e4aa173 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -27,7 +27,6 @@ static const char *PUB_TAG = "livekit_peer.pub"; typedef struct { peer_options_t options; - bool is_primary; esp_peer_role_t ice_role; esp_peer_handle_t connection; diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index b3e83b9..cfd97f4 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -35,10 +35,6 @@ typedef struct { /// Weather to force the use of relay ICE candidates. bool force_relay; - /// Whether the peer is the primary peer. - /// @note This determines which peer controls the data channels. - bool is_primary; - /// Media options used for creating SDP messages. engine_media_options_t* media; From 2c7e496a148e229c39195c071e3d6b761f5157f3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:41:43 +1000 Subject: [PATCH 07/81] Use event queue approach --- components/livekit/Kconfig | 3 + components/livekit/core/engine.c | 398 ++++++++++++++----------------- 2 files changed, 177 insertions(+), 224 deletions(-) diff --git a/components/livekit/Kconfig b/components/livekit/Kconfig index d9a2ded..d04a736 100644 --- a/components/livekit/Kconfig +++ b/components/livekit/Kconfig @@ -3,4 +3,7 @@ menu "LiveKit" int "Maximum connection retries" range 0 100 default 7 + config LK_ENGINE_QUEUE_SIZE + int "Number of engine events to queue" + default 32 endmenu diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 23c3890..edd2589 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -13,15 +13,30 @@ // MARK: - Constants static const char* TAG = "livekit_engine"; -// MARK: - Event bits -#define ENGINE_EV_CONNECT_CMD (1 << 0) -#define ENGINE_EV_CLOSE_CMD (1 << 1) -#define ENGINE_EV_COMPONENT_STATE_CHANGED (1 << 2) -#define ENGINE_EV_JOIN_RECEIVED (1 << 3) -#define ENGINE_EV_LEAVE_RECEIVED (1 << 4) - // MARK: - Type definitions +typedef enum { + EV_CMD_CONNECT, + EV_CMD_CLOSE, + EV_SIG_STATE, + EV_SIG_JOIN, + EV_SIG_LEAVE, + EV_PEER_STATE, + _EV_STATE_ENTER, + _EV_STATE_EXIT, +} engine_event_type_t; + +typedef struct { + engine_event_type_t type; + union { + struct { const char *server_url; const char *token; } cmd_connect; + struct { connection_state_t state; } sig_state; + struct { bool subscriber_primary; bool force_relay; } sig_join; + struct { livekit_pb_disconnect_reason_t reason; livekit_pb_leave_request_action_t action; } sig_leave; + struct { connection_state_t state; } peer_state; + } detail; +} engine_event_t; + typedef struct { engine_state_t state; engine_options_t options; @@ -30,50 +45,66 @@ typedef struct { peer_handle_t pub_peer_handle; // peer_handle_t sub_peer_handle; - connection_state_t signal_state; - connection_state_t pub_peer_state; - // Session state - livekit_pb_disconnect_reason_t disconnect_reason; - livekit_pb_leave_request_action_t leave_action; bool is_subscriber_primary; bool force_relay; - SemaphoreHandle_t state_mutex; char* server_url; char* token; TaskHandle_t task_handle; - EventGroupHandle_t event_group; + QueueHandle_t event_queue; bool is_running; int retry_count; } engine_t; +static void event_free(engine_event_t *ev) +{ + if (ev == NULL) return; + switch (ev->type) { + // Free event types that contain dynamic payloads + case EV_CMD_CONNECT: + free((char *)ev->detail.cmd_connect.server_url); + free((char *)ev->detail.cmd_connect.token); + break; + default: break; + } +} + // MARK: - Signal event handlers static void on_signal_state_changed(connection_state_t state, void *ctx) { engine_t *eng = (engine_t *)ctx; - eng->signal_state = state; - xEventGroupSetBits(eng->event_group, ENGINE_EV_COMPONENT_STATE_CHANGED); + engine_event_t ev = { + .type = EV_SIG_STATE, + .detail.sig_state = { .state = state } + }; + xQueueSendToFront(eng->event_queue, &ev, 0); } static void on_signal_join(livekit_pb_join_response_t *join_res, void *ctx) { engine_t *eng = (engine_t *)ctx; - eng->is_subscriber_primary = join_res->subscriber_primary; - eng->force_relay = join_res->has_client_configuration && - join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; - // TODO: Retain other fields - xEventGroupSetBits(eng->event_group, ENGINE_EV_JOIN_RECEIVED); + engine_event_t ev = { + .type = EV_SIG_JOIN, + .detail.sig_join = { + .subscriber_primary = join_res->subscriber_primary, + .force_relay = join_res->has_client_configuration && + join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED + } + }; + xQueueSendToFront(eng->event_queue, &ev, 0); } static void on_signal_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_leave_request_action_t action, void *ctx) { engine_t *eng = (engine_t *)ctx; - eng->disconnect_reason = reason; - eng->leave_action = action; - xEventGroupSetBits(eng->event_group, ENGINE_EV_LEAVE_RECEIVED); + engine_event_t ev = { + .type = EV_SIG_LEAVE, + .detail.sig_leave = { .reason = reason, .action = action } + }; + xQueueSendToFront(eng->event_queue, &ev, 0); } // MARK: - Peer event handlers @@ -93,8 +124,11 @@ static void on_peer_packet_received(livekit_pb_data_packet_t* packet, void *ctx) static void on_peer_pub_state_changed(connection_state_t state, void *ctx) { engine_t *eng = (engine_t *)ctx; - eng->pub_peer_state = state; - xEventGroupSetBits(eng->event_group, ENGINE_EV_COMPONENT_STATE_CHANGED); + engine_event_t ev = { + .type = EV_PEER_STATE, + .detail.peer_state = { .state = state } + }; + xQueueSendToFront(eng->event_queue, &ev, 0); } static void on_peer_pub_offer(const char *sdp, void *ctx) @@ -136,7 +170,6 @@ static bool connect_peers(engine_t *eng) }; // 1. Publisher peer - options.is_primary = !eng->is_subscriber_primary; options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; options.on_state_changed = on_peer_pub_state_changed; options.on_sdp = on_peer_pub_offer; @@ -150,192 +183,111 @@ static bool connect_peers(engine_t *eng) // MARK: - State machine -static void handle_state_disconnected(engine_t *eng); -static void handle_state_connecting(engine_t *eng); -static void handle_state_connected(engine_t *eng); -static void handle_state_reconnecting(engine_t *eng); -static void handle_state_disconnecting(engine_t *eng); +static bool handle_state_any(engine_t *eng, const engine_event_t *ev); +static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); +static void flush_event_queue(engine_t *eng); static void engine_task(void *arg) { engine_t *eng = (engine_t *)arg; while (eng->is_running) { - engine_state_t state = eng->state; - switch (state) { - case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng); break; - case ENGINE_STATE_CONNECTING: handle_state_connecting(eng); break; - case ENGINE_STATE_CONNECTED: handle_state_connected(eng); break; - case ENGINE_STATE_RECONNECTING: handle_state_reconnecting(eng); break; - case ENGINE_STATE_DISCONNECTING: handle_state_disconnecting(eng); break; - default: break; + engine_event_t ev; + if (!xQueueReceive(eng->event_queue, &ev, portMAX_DELAY)) { + ESP_LOGE(TAG, "Failed to receive event"); + continue; } + assert(ev.type != EV_STATE_ENTER && ev.type != EV_STATE_EXIT); + ESP_LOGI(TAG, "Event: %d", ev.type); + + engine_state_t state = eng->state; + do { + if (handle_state_any(eng, &ev)) { + break; + } + handle_state(eng, &ev, state); + } while (0); + if (eng->state != state) { ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); - // TODO: Dispatch change event - continue; + + state = eng->state; + handle_state(eng, &(engine_event_t){ .type = _EV_STATE_EXIT }, state); + assert(eng->state == state); + + handle_state(eng, &(engine_event_t){ .type = _EV_STATE_ENTER }, eng->state); + assert(eng->state == state); } - ESP_LOGI(TAG, "Re-entering state %d", eng->state); + event_free(&ev); } + flush_event_queue(eng); vTaskDelete(NULL); } -static void handle_state_disconnected(engine_t *eng) +static bool handle_state_any(engine_t *eng, const engine_event_t *ev) { - EventBits_t bits = xEventGroupWaitBits( - eng->event_group, - ENGINE_EV_CONNECT_CMD | ENGINE_EV_CLOSE_CMD, - pdTRUE, - pdFALSE, - portMAX_DELAY - ); - if (bits & ENGINE_EV_CONNECT_CMD) { - eng->state = ENGINE_STATE_CONNECTING; - } + // Handle transitions from any state here. + // Return true if the event was handled, false otherwise. + return false; } -static void handle_state_connecting(engine_t *eng) +static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { - if (xSemaphoreTake(eng->state_mutex, pdMS_TO_TICKS(100)) != pdPASS) { - eng->state = ENGINE_STATE_DISCONNECTED; - return; - } - if (signal_connect(eng->signal_handle, eng->server_url, eng->token) != SIGNAL_ERR_NONE) { - ESP_LOGE(TAG, "Failed to connect signal client"); - eng->state = ENGINE_STATE_DISCONNECTED; - xSemaphoreGive(eng->state_mutex); - return; - } - xSemaphoreGive(eng->state_mutex); - - // 1. Wait for signal connected - while (1) { - EventBits_t bits = xEventGroupWaitBits( - eng->event_group, - ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED, - pdTRUE, - pdFALSE, - portMAX_DELAY - ); - - if (bits & ENGINE_EV_CLOSE_CMD) { - eng->state = ENGINE_STATE_DISCONNECTING; - return; - } - if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { - if (eng->signal_state == CONNECTION_STATE_CONNECTED) { - break; - } - if (eng->signal_state == CONNECTION_STATE_FAILED || - eng->signal_state == CONNECTION_STATE_DISCONNECTED) { - // TODO: Check error code (4xx is user error and should go to disconnecting) - eng->state = ENGINE_STATE_RECONNECTING; - return; - } - } + if (ev->type == EV_CMD_CONNECT) { + const char *server_url = ev->detail.cmd_connect.server_url; + const char *token = ev->detail.cmd_connect.token; + + if (eng->server_url != NULL) free(eng->server_url); + if (eng->token != NULL) free(eng->token); + eng->server_url = strdup(server_url); + eng->token = strdup(token); + + eng->state = ENGINE_STATE_CONNECTING; } +} - // 2. Wait for join response - while (1) { - EventBits_t bits = xEventGroupWaitBits( - eng->event_group, - ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_JOIN_RECEIVED, - pdTRUE, - pdFALSE, - portMAX_DELAY - ); - if (bits & ENGINE_EV_CLOSE_CMD) { - eng->state = ENGINE_STATE_DISCONNECTING; - return; - } - if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { - if (eng->signal_state != CONNECTION_STATE_CONNECTED) { +static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) +{ + switch (ev->type) { + case _EV_STATE_ENTER: + signal_connect(eng->signal_handle, eng->server_url, eng->token); + break; + case EV_SIG_STATE: + // TODO: Check error code (4xx should go to disconnected) + if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || + ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_RECONNECTING; - return; } - } - if (bits & ENGINE_EV_JOIN_RECEIVED) { break; - } - } - - // 3. Start peer connections, wait for primary connected - if (!connect_peers(eng)) { - eng->state = ENGINE_STATE_DISCONNECTING; - return; - } - while (1) { - EventBits_t bits = xEventGroupWaitBits( - eng->event_group, - ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_LEAVE_RECEIVED, - pdTRUE, - pdFALSE, - portMAX_DELAY - ); - if (bits & ENGINE_EV_CLOSE_CMD) { + case EV_SIG_LEAVE: + ESP_LOGI(TAG, "Server sent leave before fully connected"); eng->state = ENGINE_STATE_DISCONNECTING; - return; - } - if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { - if (eng->signal_state != CONNECTION_STATE_CONNECTED || - eng->pub_peer_state == CONNECTION_STATE_FAILED) { - eng->state = ENGINE_STATE_RECONNECTING; - return; - } - if (eng->pub_peer_state == CONNECTION_STATE_CONNECTED) { + break; + case EV_SIG_JOIN: + if (!connect_peers(eng)) { + ESP_LOGE(TAG, "Failed to connect peers"); + eng->state = ENGINE_STATE_DISCONNECTING; break; } - } - if (bits & ENGINE_EV_LEAVE_RECEIVED) { - // TODO: Factor out this common code - switch (eng->leave_action) { - case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RECONNECT: - case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RESUME: - eng->state = ENGINE_STATE_RECONNECTING; - break; - default: - eng->state = ENGINE_STATE_DISCONNECTING; + eng->state = ENGINE_STATE_CONNECTED; + break; + case EV_PEER_STATE: + if (ev->detail.peer_state.state == CONNECTION_STATE_CONNECTED) { + eng->state = ENGINE_STATE_CONNECTED; // Fully connected + } else if (ev->detail.peer_state.state == CONNECTION_STATE_FAILED || + ev->detail.peer_state.state == CONNECTION_STATE_DISCONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; } - return; - } + break; + default: break; } - eng->state = ENGINE_STATE_CONNECTED; } -static void handle_state_connected(engine_t *eng) +static void handle_state_connected(engine_t *eng, const engine_event_t *ev) { // TODO: Track pub/sub - - EventBits_t bits = xEventGroupWaitBits( - eng->event_group, - ENGINE_EV_CLOSE_CMD | ENGINE_EV_COMPONENT_STATE_CHANGED | ENGINE_EV_LEAVE_RECEIVED, - pdTRUE, - pdFALSE, - portMAX_DELAY - ); - if (bits & ENGINE_EV_CLOSE_CMD) { - eng->state = ENGINE_STATE_DISCONNECTING; - return; - } - if (bits & ENGINE_EV_COMPONENT_STATE_CHANGED) { - if (eng->signal_state != CONNECTION_STATE_CONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; - } - return; - } - if (bits & ENGINE_EV_LEAVE_RECEIVED) { - switch (eng->leave_action) { - case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RECONNECT: - case LIVEKIT_PB_LEAVE_REQUEST_ACTION_RESUME: - eng->state = ENGINE_STATE_RECONNECTING; - break; - default: - eng->state = ENGINE_STATE_DISCONNECTING; - } - return; - } } -static void handle_state_reconnecting(engine_t *eng) +static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) { if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); @@ -343,7 +295,7 @@ static void handle_state_reconnecting(engine_t *eng) return; } - // TODO: Exponential backoff + // TODO: Use timer, exponential backoff uint32_t backoff_ms = 1000; ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu32 "ms", @@ -352,23 +304,39 @@ static void handle_state_reconnecting(engine_t *eng) vTaskDelay(pdMS_TO_TICKS(backoff_ms)); eng->retry_count++; - // TODO: Try connection, transition to connected if successful - // - Handle full vs partial reconnect + eng->state = ENGINE_STATE_CONNECTING; } -static void handle_state_disconnecting(engine_t *eng) +static void handle_state_disconnecting(engine_t *eng, const engine_event_t *ev) { - // TODO: Send leave if user initiated - disconnect_peer(&eng->pub_peer_handle); - if (eng->signal_state != CONNECTION_STATE_DISCONNECTED) { - signal_close(eng->signal_handle); - // TODO: Wait until disconnected - } + signal_close(eng->signal_handle); + // TODO: Wait for normal signal closure + flush_event_queue(eng); eng->state = ENGINE_STATE_DISCONNECTED; } +static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) +{ + switch (state) { + case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng, ev); break; + case ENGINE_STATE_CONNECTING: handle_state_connecting(eng, ev); break; + case ENGINE_STATE_CONNECTED: handle_state_connected(eng, ev); break; + case ENGINE_STATE_RECONNECTING: handle_state_reconnecting(eng, ev); break; + case ENGINE_STATE_DISCONNECTING: handle_state_disconnecting(eng, ev); break; + default: break; + } +} + +static void flush_event_queue(engine_t *eng) +{ + engine_event_t ev; + while (xQueueReceive(eng->event_queue, &ev, 0) == pdPASS) { + event_free(&ev); + } +} + // MARK: - Public API engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) @@ -377,14 +345,9 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) if (eng == NULL) { return ENGINE_ERR_NO_MEM; } - eng->state_mutex = xSemaphoreCreateMutex(); - if (eng->state_mutex == NULL) { - free(eng); - return ENGINE_ERR_NO_MEM; - } - eng->event_group = xEventGroupCreate(); - if (eng->event_group == NULL) { - vSemaphoreDelete(eng->state_mutex); + + eng->event_queue = xQueueCreate(CONFIG_LK_ENGINE_QUEUE_SIZE, sizeof(engine_event_t)); + if (eng->event_queue == NULL) { free(eng); return ENGINE_ERR_NO_MEM; } @@ -397,8 +360,7 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) // TODO: Add other handlers }; if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { - vEventGroupDelete(eng->event_group); - vSemaphoreDelete(eng->state_mutex); + free(eng->event_queue); free(eng); return ENGINE_ERR_SIGNALING; } @@ -408,8 +370,7 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) eng->is_running = true; if (xTaskCreate(engine_task, "engine_task", 4096, eng, 5, &eng->task_handle) != pdPASS) { - vEventGroupDelete(eng->event_group); - vSemaphoreDelete(eng->state_mutex); + free(eng->event_queue); free(eng); return ENGINE_ERR_NO_MEM; } @@ -434,15 +395,8 @@ engine_err_t engine_destroy(engine_handle_t handle) signal_destroy(eng->signal_handle); peer_destroy(eng->pub_peer_handle); - xSemaphoreTake(eng->state_mutex, portMAX_DELAY); - if (eng->server_url != NULL) { - free(eng->server_url); - } - if (eng->token != NULL) { - free(eng->token); - } - vSemaphoreDelete(eng->state_mutex); - vEventGroupDelete(eng->event_group); + if (eng->server_url != NULL) free(eng->server_url); + if (eng->token != NULL) free(eng->token); // TODO: Free other resources free(eng); return ENGINE_ERR_NONE; @@ -455,20 +409,13 @@ engine_err_t engine_connect(engine_handle_t handle, const char* server_url, cons } engine_t *eng = (engine_t *)handle; - if (xSemaphoreTake(eng->state_mutex, pdMS_TO_TICKS(100)) != pdPASS) { + engine_event_t ev = { + .type = EV_CMD_CONNECT, + .detail.cmd_connect = { .server_url = strdup(server_url), .token = strdup(token) } + }; + if (xQueueSendToFront(eng->event_queue, &ev, 0) != pdPASS) { return ENGINE_ERR_OTHER; } - if (eng->server_url != NULL) { - free(eng->server_url); - } - if (eng->token != NULL) { - free(eng->token); - } - eng->server_url = strdup(server_url); - eng->token = strdup(token); - xSemaphoreGive(eng->state_mutex); - - xEventGroupSetBits(eng->event_group, ENGINE_EV_CONNECT_CMD); return ENGINE_ERR_NONE; } @@ -479,7 +426,10 @@ engine_err_t engine_close(engine_handle_t handle) } engine_t *eng = (engine_t *)handle; - xEventGroupSetBits(eng->event_group, ENGINE_EV_CLOSE_CMD); + engine_event_t ev = { .type = EV_CMD_CLOSE }; + if (xQueueSendToFront(eng->event_queue, &ev, 0) != pdPASS) { + return ENGINE_ERR_OTHER; + } return ENGINE_ERR_NONE; } From 46b1403f2db040e2fdb59dfbc22a20fb9c34ba0b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:58:22 +1000 Subject: [PATCH 08/81] Introduce timer --- components/livekit/core/engine.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index edd2589..d42b90e 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -22,6 +22,7 @@ typedef enum { EV_SIG_JOIN, EV_SIG_LEAVE, EV_PEER_STATE, + EV_TIMER_EXP, _EV_STATE_ENTER, _EV_STATE_EXIT, } engine_event_type_t; @@ -54,6 +55,7 @@ typedef struct { TaskHandle_t task_handle; QueueHandle_t event_queue; + TimerHandle_t timer; bool is_running; int retry_count; } engine_t; @@ -137,6 +139,15 @@ static void on_peer_pub_offer(const char *sdp, void *ctx) // TODO: Handle offer } +// MARK: - Timer expired handler + +static void on_timer_expired(TimerHandle_t timer) +{ + engine_t *eng = (engine_t *)pvTimerGetTimerID(timer); + engine_event_t ev = { .type = EV_TIMER_EXP }; + xQueueSend(eng->event_queue, &ev, 0); +} + // MARK: - Peer lifecycle static bool disconnect_peer(peer_handle_t *peer) @@ -352,6 +363,19 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) return ENGINE_ERR_NO_MEM; } + eng->timer = xTimerCreate( + "lk_engine_timer", + pdMS_TO_TICKS(1000), + pdTRUE, + eng, + on_timer_expired); + + if (eng->timer == NULL) { + free(eng->event_queue); + free(eng); + return ENGINE_ERR_NO_MEM; + } + signal_options_t signal_options = { .ctx = eng, .on_state_changed = on_signal_state_changed, @@ -392,6 +416,10 @@ engine_err_t engine_destroy(engine_handle_t handle) vTaskDelay(pdMS_TO_TICKS(100)); } + xTimerDelete(eng->timer, 0); + vQueueDelete(eng->event_queue); + vTaskDelete(eng->task_handle); + signal_destroy(eng->signal_handle); peer_destroy(eng->pub_peer_handle); From 7586145e806a463b565e34ba05ac638a388858cb Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:02:42 +1000 Subject: [PATCH 09/81] Proper cleanup --- components/livekit/core/engine.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index d42b90e..0de5a8c 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -371,8 +371,8 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) on_timer_expired); if (eng->timer == NULL) { - free(eng->event_queue); free(eng); + free(eng->event_queue); return ENGINE_ERR_NO_MEM; } @@ -384,8 +384,9 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) // TODO: Add other handlers }; if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { - free(eng->event_queue); free(eng); + free(eng->event_queue); + free(eng->timer); return ENGINE_ERR_SIGNALING; } @@ -394,8 +395,9 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) eng->is_running = true; if (xTaskCreate(engine_task, "engine_task", 4096, eng, 5, &eng->task_handle) != pdPASS) { - free(eng->event_queue); free(eng); + free(eng->event_queue); + free(eng->timer); return ENGINE_ERR_NO_MEM; } @@ -415,13 +417,15 @@ engine_err_t engine_destroy(engine_handle_t handle) // TODO: Wait for disconnected state or timeout vTaskDelay(pdMS_TO_TICKS(100)); } + vTaskDelete(eng->task_handle); - xTimerDelete(eng->timer, 0); + xTimerDelete(eng->timer, portMAX_DELAY); vQueueDelete(eng->event_queue); - vTaskDelete(eng->task_handle); signal_destroy(eng->signal_handle); - peer_destroy(eng->pub_peer_handle); + if (eng->pub_peer_handle != NULL) { + peer_destroy(eng->pub_peer_handle); + } if (eng->server_url != NULL) free(eng->server_url); if (eng->token != NULL) free(eng->token); From eafe3f154b9759660e52428fb0d95dd3bd0f94cd Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:15:02 +1000 Subject: [PATCH 10/81] Implement reconnecting state --- components/livekit/core/engine.c | 44 ++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 0de5a8c..7354aa2 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -23,6 +23,7 @@ typedef enum { EV_SIG_LEAVE, EV_PEER_STATE, EV_TIMER_EXP, + EV_MAX_RETRIES_REACHED, _EV_STATE_ENTER, _EV_STATE_EXIT, } engine_event_type_t; @@ -300,29 +301,40 @@ static void handle_state_connected(engine_t *eng, const engine_event_t *ev) static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) { - if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { - ESP_LOGW(TAG, "Max retries reached"); - eng->state = ENGINE_STATE_DISCONNECTED; - return; - } - - // TODO: Use timer, exponential backoff - uint32_t backoff_ms = 1000; - - ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu32 "ms", - eng->retry_count + 1, CONFIG_LK_MAX_RETRIES, backoff_ms); + switch (ev->type) { + case _EV_STATE_ENTER: + if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { + ESP_LOGW(TAG, "Max retries reached"); + xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); + break; + } + // TODO: Exponential backoff + uint32_t backoff_ms = 1000; - vTaskDelay(pdMS_TO_TICKS(backoff_ms)); - eng->retry_count++; + ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu32 "ms", + eng->retry_count + 1, CONFIG_LK_MAX_RETRIES, backoff_ms); - eng->state = ENGINE_STATE_CONNECTING; + xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); + xTimerStart(eng->timer, 0); + break; + case EV_MAX_RETRIES_REACHED: + eng->state = ENGINE_STATE_DISCONNECTED; + break; + case EV_TIMER_EXP: + eng->retry_count++; + eng->state = ENGINE_STATE_CONNECTING; + break; + case _EV_STATE_EXIT: + xTimerStop(eng->timer, portMAX_DELAY); + break; + default: break; + } } static void handle_state_disconnecting(engine_t *eng, const engine_event_t *ev) { disconnect_peer(&eng->pub_peer_handle); - signal_close(eng->signal_handle); // TODO: Wait for normal signal closure flush_event_queue(eng); eng->state = ENGINE_STATE_DISCONNECTED; @@ -366,7 +378,7 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) eng->timer = xTimerCreate( "lk_engine_timer", pdMS_TO_TICKS(1000), - pdTRUE, + pdFALSE, eng, on_timer_expired); From bc90f795d1898d773f3a06c70a7c17093a42a08e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:58:21 +1000 Subject: [PATCH 11/81] Remove disconnecting state --- components/livekit/core/engine.c | 14 ++------------ components/livekit/core/engine.h | 3 +-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 7354aa2..aa37d5a 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -272,12 +272,12 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) break; case EV_SIG_LEAVE: ESP_LOGI(TAG, "Server sent leave before fully connected"); - eng->state = ENGINE_STATE_DISCONNECTING; + eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_SIG_JOIN: if (!connect_peers(eng)) { ESP_LOGE(TAG, "Failed to connect peers"); - eng->state = ENGINE_STATE_DISCONNECTING; + eng->state = ENGINE_STATE_DISCONNECTED; break; } eng->state = ENGINE_STATE_CONNECTED; @@ -331,15 +331,6 @@ static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) } } -static void handle_state_disconnecting(engine_t *eng, const engine_event_t *ev) -{ - disconnect_peer(&eng->pub_peer_handle); - - // TODO: Wait for normal signal closure - flush_event_queue(eng); - eng->state = ENGINE_STATE_DISCONNECTED; -} - static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) { switch (state) { @@ -347,7 +338,6 @@ static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state case ENGINE_STATE_CONNECTING: handle_state_connecting(eng, ev); break; case ENGINE_STATE_CONNECTED: handle_state_connected(eng, ev); break; case ENGINE_STATE_RECONNECTING: handle_state_reconnecting(eng, ev); break; - case ENGINE_STATE_DISCONNECTING: handle_state_disconnecting(eng, ev); break; default: break; } } diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index d3d1c72..8d9f60b 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -22,8 +22,7 @@ typedef enum { ENGINE_STATE_DISCONNECTED, ENGINE_STATE_CONNECTING, ENGINE_STATE_CONNECTED, - ENGINE_STATE_RECONNECTING, - ENGINE_STATE_DISCONNECTING + ENGINE_STATE_RECONNECTING } engine_state_t; typedef enum { From cd12ec366ba85540322d033b2be2210e47f7c2fc Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:16:36 +1000 Subject: [PATCH 12/81] Reintroduce all features --- components/livekit/Kconfig | 9 + components/livekit/core/engine.c | 520 +++++++++++++++++++++++++++---- 2 files changed, 470 insertions(+), 59 deletions(-) diff --git a/components/livekit/Kconfig b/components/livekit/Kconfig index d04a736..676c717 100644 --- a/components/livekit/Kconfig +++ b/components/livekit/Kconfig @@ -6,4 +6,13 @@ menu "LiveKit" config LK_ENGINE_QUEUE_SIZE int "Number of engine events to queue" default 32 + config LK_PUB_INTERVAL_MS + int "How often to capture and send AV frames" + default 20 + config LK_PUB_AUDIO_TRACK_NAME + string "Name of the published audio track" + default "Audio" + config LK_PUB_VIDEO_TRACK_NAME + string "Name of the published video track" + default "Video" endmenu diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index aa37d5a..da2013f 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -2,6 +2,8 @@ #include "freertos/task.h" #include "freertos/semphr.h" #include "freertos/event_groups.h" +#include "media_lib_os.h" +#include "esp_codec_dev.h" #include #include #include "esp_log.h" @@ -15,25 +17,28 @@ static const char* TAG = "livekit_engine"; // MARK: - Type definitions +/// Type of event processed by the engine state machine. typedef enum { EV_CMD_CONNECT, EV_CMD_CLOSE, EV_SIG_STATE, EV_SIG_JOIN, EV_SIG_LEAVE, - EV_PEER_STATE, + EV_PEER_PUB_STATE, + EV_PEER_SUB_STATE, EV_TIMER_EXP, EV_MAX_RETRIES_REACHED, _EV_STATE_ENTER, _EV_STATE_EXIT, } engine_event_type_t; +/// An event processed by the engine state machine. typedef struct { engine_event_type_t type; union { struct { const char *server_url; const char *token; } cmd_connect; struct { connection_state_t state; } sig_state; - struct { bool subscriber_primary; bool force_relay; } sig_join; + struct { bool subscriber_primary; bool force_relay; char local_participant_sid[32]; } sig_join; struct { livekit_pb_disconnect_reason_t reason; livekit_pb_leave_request_action_t action; } sig_leave; struct { connection_state_t state; } peer_state; } detail; @@ -45,7 +50,12 @@ typedef struct { signal_handle_t signal_handle; peer_handle_t pub_peer_handle; - // peer_handle_t sub_peer_handle; + peer_handle_t sub_peer_handle; + + esp_capture_path_handle_t capturer_path; + bool is_media_streaming; + + esp_codec_dev_handle_t renderer_handle; // Session state bool is_subscriber_primary; @@ -61,11 +71,11 @@ typedef struct { int retry_count; } engine_t; +/// Frees engine events that contain dynamic payloads. static void event_free(engine_event_t *ev) { if (ev == NULL) return; switch (ev->type) { - // Free event types that contain dynamic payloads case EV_CMD_CONNECT: free((char *)ev->detail.cmd_connect.server_url); free((char *)ev->detail.cmd_connect.token); @@ -74,6 +84,211 @@ static void event_free(engine_event_t *ev) } } +// MARK: - Subscribed media + +/// Converts `esp_peer_audio_codec_t` to equivalent `av_render_audio_codec_t` value. +static inline av_render_audio_codec_t get_dec_codec(esp_peer_audio_codec_t codec) +{ + switch (codec) { + case ESP_PEER_AUDIO_CODEC_G711A: return AV_RENDER_AUDIO_CODEC_G711A; + case ESP_PEER_AUDIO_CODEC_G711U: return AV_RENDER_AUDIO_CODEC_G711U; + case ESP_PEER_AUDIO_CODEC_OPUS: return AV_RENDER_AUDIO_CODEC_OPUS; + default: return AV_RENDER_AUDIO_CODEC_NONE; + } +} + +/// Maps `esp_peer_audio_stream_info_t` to `av_render_audio_info_t`. +static inline void convert_dec_aud_info(esp_peer_audio_stream_info_t *info, av_render_audio_info_t *dec_info) +{ + dec_info->codec = get_dec_codec(info->codec); + if (info->codec == ESP_PEER_AUDIO_CODEC_G711A || info->codec == ESP_PEER_AUDIO_CODEC_G711U) { + dec_info->sample_rate = 8000; + dec_info->channel = 1; + } else { + dec_info->sample_rate = info->sample_rate; + dec_info->channel = info->channel; + } + dec_info->bits_per_sample = 16; +} + +// MARK: - Published media + +/// Converts `esp_peer_audio_codec_t` to equivalent `esp_capture_codec_type_t` value. +static inline esp_capture_codec_type_t capture_audio_codec_type(esp_peer_audio_codec_t peer_codec) +{ + switch (peer_codec) { + case ESP_PEER_AUDIO_CODEC_G711A: return ESP_CAPTURE_CODEC_TYPE_G711A; + case ESP_PEER_AUDIO_CODEC_G711U: return ESP_CAPTURE_CODEC_TYPE_G711U; + case ESP_PEER_AUDIO_CODEC_OPUS: return ESP_CAPTURE_CODEC_TYPE_OPUS; + default: return ESP_CAPTURE_CODEC_TYPE_NONE; + } +} + +/// Converts `esp_peer_video_codec_t` to equivalent `esp_capture_codec_type_t` value. +static inline esp_capture_codec_type_t capture_video_codec_type(esp_peer_video_codec_t peer_codec) +{ + switch (peer_codec) { + case ESP_PEER_VIDEO_CODEC_H264: return ESP_CAPTURE_CODEC_TYPE_H264; + case ESP_PEER_VIDEO_CODEC_MJPEG: return ESP_CAPTURE_CODEC_TYPE_MJPEG; + default: return ESP_CAPTURE_CODEC_TYPE_NONE; + } +} + +/// Captures and sends a single audio frame over the peer connection. +__attribute__((always_inline)) +static inline void _media_stream_send_audio(engine_t *eng) +{ + esp_capture_stream_frame_t audio_frame = { + .stream_type = ESP_CAPTURE_STREAM_TYPE_AUDIO, + }; + while (esp_capture_acquire_path_frame(eng->capturer_path, &audio_frame, true) == ESP_CAPTURE_ERR_OK) { + esp_peer_audio_frame_t audio_send_frame = { + .pts = audio_frame.pts, + .data = audio_frame.data, + .size = audio_frame.size, + }; + peer_send_audio(eng->pub_peer_handle, &audio_send_frame); + esp_capture_release_path_frame(eng->capturer_path, &audio_frame); + } +} + +/// Captures and sends a single video frame over the peer connection. +__attribute__((always_inline)) +static inline void _media_stream_send_video(engine_t *eng) +{ + esp_capture_stream_frame_t video_frame = { + .stream_type = ESP_CAPTURE_STREAM_TYPE_VIDEO, + }; + if (esp_capture_acquire_path_frame(eng->capturer_path, &video_frame, true) == ESP_CAPTURE_ERR_OK) { + esp_peer_video_frame_t video_send_frame = { + .pts = video_frame.pts, + .data = video_frame.data, + .size = video_frame.size, + }; + peer_send_video(eng->pub_peer_handle, &video_send_frame); + esp_capture_release_path_frame(eng->capturer_path, &video_frame); + } +} + +static void media_stream_task(void *arg) +{ + engine_t *eng = (engine_t *)arg; + while (eng->is_media_streaming) { + if (eng->options.media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE) { + _media_stream_send_audio(eng); + } + if (eng->options.media.video_info.codec != ESP_PEER_VIDEO_CODEC_NONE) { + _media_stream_send_video(eng); + } + media_lib_thread_sleep(CONFIG_LK_PUB_INTERVAL_MS); + } + media_lib_thread_destroy(NULL); +} + +static engine_err_t media_stream_begin(engine_t *eng) +{ + if (esp_capture_start(eng->options.media.capturer) != ESP_CAPTURE_ERR_OK) { + ESP_LOGE(TAG, "Failed to start capture"); + return ENGINE_ERR_MEDIA; + } + media_lib_thread_handle_t handle = NULL; + eng->is_media_streaming = true; + if (media_lib_thread_create_from_scheduler(&handle, STREAM_THREAD_NAME, media_stream_task, eng) != ESP_OK) { + ESP_LOGE(TAG, "Failed to create media stream thread"); + eng->is_media_streaming = false; + return ENGINE_ERR_MEDIA; + } + return ENGINE_ERR_NONE; +} + +static engine_err_t media_stream_end(engine_t *eng) +{ + if (!eng->is_media_streaming) { + return ENGINE_ERR_NONE; + } + eng->is_media_streaming = false; + esp_capture_stop(eng->options.media.capturer); + return ENGINE_ERR_NONE; +} + +static engine_err_t send_add_audio_track(engine_t *eng) +{ + bool is_stereo = eng->options.media.audio_info.channel == 2; + livekit_pb_add_track_request_t req = { + .cid = "a0", + .name = CONFIG_LK_PUB_AUDIO_TRACK_NAME, + .type = LIVEKIT_PB_TRACK_TYPE_AUDIO, + .source = LIVEKIT_PB_TRACK_SOURCE_MICROPHONE, + .muted = false, + .audio_features_count = is_stereo ? 1 : 0, + .audio_features = { LIVEKIT_PB_AUDIO_TRACK_FEATURE_TF_STEREO }, + .layers_count = 0 + }; + + if (signal_send_add_track(eng->signal_handle, &req) != SIGNAL_ERR_NONE) { + ESP_LOGE(TAG, "Failed to publish audio track"); + return ENGINE_ERR_SIGNALING; + } + return ENGINE_ERR_NONE; +} + +static engine_err_t send_add_video_track(engine_t *eng) +{ + livekit_pb_video_layer_t video_layer = { + .quality = LIVEKIT_PB_VIDEO_QUALITY_HIGH, + .width = eng->options.media.video_info.width, + .height = eng->options.media.video_info.height + }; + livekit_pb_add_track_request_t req = { + .cid = "v0", + .name = CONFIG_LK_PUB_VIDEO_TRACK_NAME, + .type = LIVEKIT_PB_TRACK_TYPE_VIDEO, + .source = LIVEKIT_PB_TRACK_SOURCE_CAMERA, + .muted = false, + .layers_count = 1, + .layers = { video_layer }, + .audio_features_count = 0 + }; + + if (signal_send_add_track(eng->signal_handle, &req) != SIGNAL_ERR_NONE) { + ESP_LOGE(TAG, "Failed to publish video track"); + return ENGINE_ERR_SIGNALING; + } + return ENGINE_ERR_NONE; +} + +/// Begins media streaming and sends add track requests. +static engine_err_t publish_tracks(engine_t *eng) +{ + if (eng->options.media.audio_info.codec == ESP_PEER_AUDIO_CODEC_NONE && + eng->options.media.video_info.codec == ESP_PEER_VIDEO_CODEC_NONE) { + ESP_LOGI(TAG, "No media tracks to publish"); + return ENGINE_ERR_NONE; + } + + int ret = ENGINE_ERR_OTHER; + do { + if (media_stream_begin(eng) != ENGINE_ERR_NONE) { + ret = ENGINE_ERR_MEDIA; + break; + } + if (eng->options.media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE && + send_add_audio_track(eng) != ENGINE_ERR_NONE) { + ret = ENGINE_ERR_SIGNALING; + break; + } + if (eng->options.media.video_info.codec != ESP_PEER_VIDEO_CODEC_NONE && + send_add_video_track(eng) != ENGINE_ERR_NONE) { + ret = ENGINE_ERR_SIGNALING; + break; + } + return ENGINE_ERR_NONE; + } while (0); + + media_stream_end(eng); + return ret; +} + // MARK: - Signal event handlers static void on_signal_state_changed(connection_state_t state, void *ctx) @@ -110,25 +325,69 @@ static void on_signal_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_le xQueueSendToFront(eng->event_queue, &ev, 0); } -// MARK: - Peer event handlers +static void on_signal_answer(const char *sdp, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + peer_handle_sdp(eng->pub_peer_handle, sdp); +} + +static void on_signal_offer(const char *sdp, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + peer_handle_sdp(eng->sub_peer_handle, sdp); +} -static void on_peer_ice_candidate(const char *candidate, void *ctx) +static void on_signal_trickle(const char *ice_candidate, livekit_pb_signal_target_t target, void *ctx) { - ESP_LOGI(TAG, "ICE candidate"); - // TODO: Handle ICE candidate + engine_t *eng = (engine_t *)ctx; + peer_handle_t target_peer = target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? + eng->sub_peer_handle : eng->pub_peer_handle; + peer_handle_ice_candidate(target_peer, ice_candidate); } +static void on_signal_room_update(const livekit_pb_room_t* info, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + if (eng->options.on_room_info) { + eng->options.on_room_info(info, eng->options.ctx); + } +} + +static void on_signal_participant_update(const livekit_pb_participant_info_t* info, void *ctx) +{ + // engine_t *eng = (engine_t *)ctx; + // TODO: Subscribe + // bool is_local = strncmp(info->sid, eng->local_participant_sid, sizeof(eng->local_participant_sid)) == 0; + // if (eng->options.on_participant_info) { + // eng->options.on_participant_info(info, is_local, eng->options.ctx); + //} + + // if (is_local) return; + // subscribe_tracks(eng, info->tracks, info->tracks_count); +} + +// MARK: - Common peer event handlers + static void on_peer_packet_received(livekit_pb_data_packet_t* packet, void *ctx) { - ESP_LOGI(TAG, "Packet received"); - // TODO: Handle packet + engine_t *eng = (engine_t *)ctx; + if (eng->options.on_data_packet) { + eng->options.on_data_packet(packet, eng->options.ctx); + } } +static void on_peer_ice_candidate(const char *candidate, void *ctx) +{ + // TODO: Handle ICE candidate +} + +// MARK: - Publisher peer event handlers + static void on_peer_pub_state_changed(connection_state_t state, void *ctx) { engine_t *eng = (engine_t *)ctx; engine_event_t ev = { - .type = EV_PEER_STATE, + .type = EV_PEER_PUB_STATE, .detail.peer_state = { .state = state } }; xQueueSendToFront(eng->event_queue, &ev, 0); @@ -136,8 +395,55 @@ static void on_peer_pub_state_changed(connection_state_t state, void *ctx) static void on_peer_pub_offer(const char *sdp, void *ctx) { - ESP_LOGI(TAG, "Publisher peer offer"); - // TODO: Handle offer + engine_t *eng = (engine_t *)ctx; + signal_send_offer(eng->signal_handle, sdp); +} + +// MARK: - Subscriber peer event handlers + +static void on_peer_sub_state_changed(connection_state_t state, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + engine_event_t ev = { + .type = EV_PEER_SUB_STATE, + .detail.peer_state = { .state = state } + }; + xQueueSendToFront(eng->event_queue, &ev, 0); +} + +static void on_peer_sub_answer(const char *sdp, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + signal_send_answer(eng->signal_handle, sdp); +} + +static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + if (eng->state != ENGINE_STATE_CONNECTED) return; + + av_render_audio_info_t render_info = {}; + convert_dec_aud_info(info, &render_info); + ESP_LOGD(TAG, "Audio render info: codec=%d, sample_rate=%" PRIu32 ", channels=%" PRIu8, + render_info.codec, render_info.sample_rate, render_info.channel); + + if (av_render_add_audio_stream(eng->renderer_handle, &render_info) != ESP_MEDIA_ERR_OK) { + ESP_LOGE(TAG, "Failed to add audio stream to renderer"); + return; + } +} + +static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + if (eng->state != ENGINE_STATE_CONNECTED) return; + + av_render_audio_data_t audio_data = { + .pts = frame->pts, + .data = frame->data, + .size = frame->size, + }; + av_render_add_audio_data(eng->renderer_handle, &audio_data); } // MARK: - Timer expired handler @@ -151,49 +457,65 @@ static void on_timer_expired(TimerHandle_t timer) // MARK: - Peer lifecycle -static bool disconnect_peer(peer_handle_t *peer) +static inline void _create_and_connect_peer(peer_options_t *options, peer_handle_t *peer) +{ + if (peer_create(peer, options) != PEER_ERR_NONE) + return; + if (peer_connect(*peer) != PEER_ERR_NONE) { + peer_destroy(*peer); + *peer = NULL; + } +} + +static inline void _disconnect_and_destroy_peer(peer_handle_t *peer) { - if (*peer == NULL) return false; - if (peer_disconnect(*peer) != PEER_ERR_NONE) return false; - if (peer_destroy(*peer) != PEER_ERR_NONE) return false; + if (!peer || !*peer) return; + peer_disconnect(*peer); + peer_destroy(*peer); *peer = NULL; - return true; } -static bool connect_peer(engine_t *eng, peer_options_t *options, peer_handle_t *peer) +static void destroy_peer_connections(engine_t *eng) { - disconnect_peer(peer); - if (peer_create(peer, options) != PEER_ERR_NONE) return false; - if (peer_connect(*peer) != PEER_ERR_NONE) return false; - return true; + _disconnect_and_destroy_peer(&eng->pub_peer_handle); + _disconnect_and_destroy_peer(&eng->sub_peer_handle); } -static bool connect_peers(engine_t *eng) +static bool establish_peer_connections(engine_t *eng) { peer_options_t options = { - // Options common to both peers - .force_relay = eng->force_relay, - .media = &eng->options.media, - // .server_list = eng->ice_servers, - //.server_count = eng->ice_server_count, - .on_ice_candidate = on_peer_ice_candidate, + .force_relay = eng->force_relay, + .media = &eng->options.media, + .on_ice_candidate = on_peer_ice_candidate, .on_packet_received = on_peer_packet_received, - .ctx = eng + .ctx = eng }; - // 1. Publisher peer - options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; + // Publisher + options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; options.on_state_changed = on_peer_pub_state_changed; - options.on_sdp = on_peer_pub_offer; + options.on_sdp = on_peer_pub_offer; + + _create_and_connect_peer(&options, &eng->pub_peer_handle); + if (eng->pub_peer_handle == NULL) + return false; + + // Subscriber + options.target = LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER; + options.on_state_changed = on_peer_sub_state_changed; + options.on_sdp = on_peer_sub_answer; + options.on_audio_info = on_peer_sub_audio_info; + options.on_audio_frame = on_peer_sub_audio_frame; - if (!connect_peer(eng, &options, &eng->pub_peer_handle)) { - ESP_LOGE(TAG, "Failed to connect publisher peer"); + _create_and_connect_peer(&options, &eng->sub_peer_handle); + if (eng->sub_peer_handle == NULL) { + _disconnect_and_destroy_peer(&eng->pub_peer_handle); return false; } return true; } -// MARK: - State machine +// MARK: - Connection state machine static bool handle_state_any(engine_t *eng, const engine_event_t *ev); static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); @@ -208,7 +530,7 @@ static void engine_task(void *arg) ESP_LOGE(TAG, "Failed to receive event"); continue; } - assert(ev.type != EV_STATE_ENTER && ev.type != EV_STATE_EXIT); + assert(ev.type != _EV_STATE_ENTER && ev.type != _EV_STATE_EXIT); ESP_LOGI(TAG, "Event: %d", ev.type); engine_state_t state = eng->state; @@ -244,16 +566,25 @@ static bool handle_state_any(engine_t *eng, const engine_event_t *ev) static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { - if (ev->type == EV_CMD_CONNECT) { - const char *server_url = ev->detail.cmd_connect.server_url; - const char *token = ev->detail.cmd_connect.token; + switch (ev->type) { + case _EV_STATE_ENTER: + media_stream_end(eng); + signal_close(eng->signal_handle); + destroy_peer_connections(eng); + eng->retry_count = 0; + break; + case EV_CMD_CONNECT: + const char *server_url = ev->detail.cmd_connect.server_url; + const char *token = ev->detail.cmd_connect.token; - if (eng->server_url != NULL) free(eng->server_url); - if (eng->token != NULL) free(eng->token); - eng->server_url = strdup(server_url); - eng->token = strdup(token); + if (eng->server_url != NULL) free(eng->server_url); + if (eng->token != NULL) free(eng->token); + eng->server_url = strdup(server_url); + eng->token = strdup(token); - eng->state = ENGINE_STATE_CONNECTING; + eng->state = ENGINE_STATE_CONNECTING; + break; + default: break; } } @@ -275,18 +606,30 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_SIG_JOIN: - if (!connect_peers(eng)) { - ESP_LOGE(TAG, "Failed to connect peers"); + eng->is_subscriber_primary = ev->detail.sig_join.subscriber_primary; + eng->force_relay = ev->detail.sig_join.force_relay; + + if (!establish_peer_connections(eng)) { + ESP_LOGE(TAG, "Failed to establish peer connections"); eng->state = ENGINE_STATE_DISCONNECTED; break; } - eng->state = ENGINE_STATE_CONNECTED; break; - case EV_PEER_STATE: - if (ev->detail.peer_state.state == CONNECTION_STATE_CONNECTED) { - eng->state = ENGINE_STATE_CONNECTED; // Fully connected - } else if (ev->detail.peer_state.state == CONNECTION_STATE_FAILED || - ev->detail.peer_state.state == CONNECTION_STATE_DISCONNECTED) { + case EV_PEER_PUB_STATE: + connection_state_t pub_state = ev->detail.peer_state.state; + if (!eng->is_subscriber_primary && pub_state == CONNECTION_STATE_CONNECTED) { + eng->state = ENGINE_STATE_CONNECTED; + } else if (pub_state == CONNECTION_STATE_FAILED || + pub_state == CONNECTION_STATE_DISCONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + } + break; + case EV_PEER_SUB_STATE: + connection_state_t sub_state = ev->detail.peer_state.state; + if (eng->is_subscriber_primary && sub_state == CONNECTION_STATE_CONNECTED) { + eng->state = ENGINE_STATE_CONNECTED; + } else if (sub_state == CONNECTION_STATE_FAILED || + sub_state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_RECONNECTING; } break; @@ -296,13 +639,43 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) static void handle_state_connected(engine_t *eng, const engine_event_t *ev) { - // TODO: Track pub/sub + switch (ev->type) { + case _EV_STATE_ENTER: + eng->retry_count = 0; + publish_tracks(eng); + break; + case EV_SIG_STATE: + if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || + ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + } + break; + case EV_PEER_PUB_STATE: + connection_state_t pub_state = ev->detail.peer_state.state; + if (pub_state == CONNECTION_STATE_FAILED || + pub_state == CONNECTION_STATE_DISCONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + } + break; + case EV_PEER_SUB_STATE: + connection_state_t sub_state = ev->detail.peer_state.state; + if (sub_state == CONNECTION_STATE_FAILED || + sub_state == CONNECTION_STATE_DISCONNECTED) { + eng->state = ENGINE_STATE_RECONNECTING; + } + break; + default: break; + } } static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: + media_stream_end(eng); + signal_close(eng->signal_handle); + destroy_peer_connections(eng); + if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); @@ -345,9 +718,12 @@ static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state static void flush_event_queue(engine_t *eng) { engine_event_t ev; + int count = 0; while (xQueueReceive(eng->event_queue, &ev, 0) == pdPASS) { + count++; event_free(&ev); } + ESP_LOGI(TAG, "Flushed %d events", count); } // MARK: - Public API @@ -373,8 +749,8 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) on_timer_expired); if (eng->timer == NULL) { - free(eng); free(eng->event_queue); + free(eng); return ENGINE_ERR_NO_MEM; } @@ -383,12 +759,16 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) .on_state_changed = on_signal_state_changed, .on_join = on_signal_join, .on_leave = on_signal_leave, - // TODO: Add other handlers + .on_answer = on_signal_answer, + .on_offer = on_signal_offer, + .on_trickle = on_signal_trickle, + .on_room_update = on_signal_room_update, + .on_participant_update = on_signal_participant_update }; if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { - free(eng); free(eng->event_queue); free(eng->timer); + free(eng); return ENGINE_ERR_SIGNALING; } @@ -397,12 +777,34 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) eng->is_running = true; if (xTaskCreate(engine_task, "engine_task", 4096, eng, 5, &eng->task_handle) != pdPASS) { - free(eng); free(eng->event_queue); free(eng->timer); + free(eng); return ENGINE_ERR_NO_MEM; } + esp_capture_sink_cfg_t sink_cfg = { + .audio_info = { + .codec = capture_audio_codec_type(eng->options.media.audio_info.codec), + .sample_rate = eng->options.media.audio_info.sample_rate, + .channel = eng->options.media.audio_info.channel, + .bits_per_sample = 16, + }, + .video_info = { + .codec = capture_video_codec_type(eng->options.media.video_info.codec), + .width = eng->options.media.video_info.width, + .height = eng->options.media.video_info.height, + .fps = eng->options.media.video_info.fps, + }, + }; + + if (options->media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE) { + // TODO: Can we ensure the renderer is valid? If not, return error. + eng->renderer_handle = options->media.renderer; + } + esp_capture_setup_path(eng->options.media.capturer, ESP_CAPTURE_PATH_PRIMARY, &sink_cfg, &eng->capturer_path); + esp_capture_enable_path(eng->capturer_path, ESP_CAPTURE_RUN_TYPE_ALWAYS); + *handle = (engine_handle_t)eng; return ENGINE_ERR_NONE; } From 894b199ba05bd4918af8bf47038f82e4f2098247 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 13:28:59 +1000 Subject: [PATCH 13/81] Add additional transitions --- components/livekit/core/engine.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index da2013f..d2e3fdf 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -568,6 +568,7 @@ static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: + // Clean up resources from previous connection (if any) media_stream_end(eng); signal_close(eng->signal_handle); destroy_peer_connections(eng); @@ -594,6 +595,13 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) case _EV_STATE_ENTER: signal_connect(eng->signal_handle, eng->server_url, eng->token); break; + case EV_CMD_CLOSE: + // TODO: Send leave request + eng->state = ENGINE_STATE_DISCONNECTED; + break; + case EV_CMD_CONNECT: + ESP_LOGW(TAG, "Engine already connecting, ignoring connect command"); + break; case EV_SIG_STATE: // TODO: Check error code (4xx should go to disconnected) if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || @@ -644,12 +652,23 @@ static void handle_state_connected(engine_t *eng, const engine_event_t *ev) eng->retry_count = 0; publish_tracks(eng); break; + case EV_CMD_CLOSE: + // TODO: Send leave request + eng->state = ENGINE_STATE_DISCONNECTED; + break; + case EV_CMD_CONNECT: + ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); + break; case EV_SIG_STATE: if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_RECONNECTING; } break; + case EV_SIG_LEAVE: + ESP_LOGI(TAG, "Server initiated disconnect"); + eng->state = ENGINE_STATE_DISCONNECTED; + break; case EV_PEER_PUB_STATE: connection_state_t pub_state = ev->detail.peer_state.state; if (pub_state == CONNECTION_STATE_FAILED || From 3e90462de0cc0aedc2b3375dd138c7bbac6cc2b1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 13:42:17 +1000 Subject: [PATCH 14/81] Docs and organization --- components/livekit/core/engine.c | 59 ++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index d2e3fdf..c117f00 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -19,28 +19,52 @@ static const char* TAG = "livekit_engine"; /// Type of event processed by the engine state machine. typedef enum { - EV_CMD_CONNECT, - EV_CMD_CLOSE, - EV_SIG_STATE, - EV_SIG_JOIN, - EV_SIG_LEAVE, - EV_PEER_PUB_STATE, - EV_PEER_SUB_STATE, - EV_TIMER_EXP, - EV_MAX_RETRIES_REACHED, + EV_CMD_CONNECT, /// User-initiated connect. + EV_CMD_CLOSE, /// User-initiated disconnect. + EV_SIG_STATE, /// Signal state changed. + EV_SIG_JOIN, /// Join response received. + EV_SIG_LEAVE, /// Leave request received. + EV_PEER_PUB_STATE, /// Publisher peer state changed. + EV_PEER_SUB_STATE, /// Subscriber peer state changed. + EV_TIMER_EXP, /// Timer expired. + EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. _EV_STATE_ENTER, _EV_STATE_EXIT, } engine_event_type_t; /// An event processed by the engine state machine. typedef struct { + /// Type of event, determines which union member is valid in `detail`. engine_event_type_t type; union { - struct { const char *server_url; const char *token; } cmd_connect; - struct { connection_state_t state; } sig_state; - struct { bool subscriber_primary; bool force_relay; char local_participant_sid[32]; } sig_join; - struct { livekit_pb_disconnect_reason_t reason; livekit_pb_leave_request_action_t action; } sig_leave; - struct { connection_state_t state; } peer_state; + /// Detail for `EV_CMD_CONNECT`. + struct { + const char *server_url; + const char *token; + } cmd_connect; + + /// Detail for `EV_SIG_STATE`. + struct { + connection_state_t state; + } sig_state; + + /// Detail for `EV_SIG_JOIN`. + struct { + bool subscriber_primary; + bool force_relay; + char local_participant_sid[32]; + } sig_join; + + /// Detail for `EV_SIG_LEAVE`. + struct { + livekit_pb_disconnect_reason_t reason; + livekit_pb_leave_request_action_t action; + } sig_leave; + + /// Detail for `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. + struct { + connection_state_t state; + } peer_state; } detail; } engine_event_t; @@ -52,15 +76,13 @@ typedef struct { peer_handle_t pub_peer_handle; peer_handle_t sub_peer_handle; + esp_codec_dev_handle_t renderer_handle; esp_capture_path_handle_t capturer_path; bool is_media_streaming; - esp_codec_dev_handle_t renderer_handle; - // Session state bool is_subscriber_primary; bool force_relay; - char* server_url; char* token; @@ -71,7 +93,7 @@ typedef struct { int retry_count; } engine_t; -/// Frees engine events that contain dynamic payloads. +/// Frees engine events that contain dynamic fields. static void event_free(engine_event_t *ev) { if (ev == NULL) return; @@ -667,6 +689,7 @@ static void handle_state_connected(engine_t *eng, const engine_event_t *ev) break; case EV_SIG_LEAVE: ESP_LOGI(TAG, "Server initiated disconnect"); + // TODO: Handle leave action eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_PEER_PUB_STATE: From eef3f49056e34bbe45d5daddfd86f25c4a21d0ce Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:24:23 +1000 Subject: [PATCH 15/81] Remove any state handler --- components/livekit/core/engine.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index c117f00..cd4531e 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -539,7 +539,6 @@ static bool establish_peer_connections(engine_t *eng) // MARK: - Connection state machine -static bool handle_state_any(engine_t *eng, const engine_event_t *ev); static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); static void flush_event_queue(engine_t *eng); @@ -556,12 +555,7 @@ static void engine_task(void *arg) ESP_LOGI(TAG, "Event: %d", ev.type); engine_state_t state = eng->state; - do { - if (handle_state_any(eng, &ev)) { - break; - } - handle_state(eng, &ev, state); - } while (0); + handle_state(eng, &ev, state); if (eng->state != state) { ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); @@ -579,13 +573,6 @@ static void engine_task(void *arg) vTaskDelete(NULL); } -static bool handle_state_any(engine_t *eng, const engine_event_t *ev) -{ - // Handle transitions from any state here. - // Return true if the event was handled, false otherwise. - return false; -} - static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { From 85d8e94fbfef9027e6446564f29b127da05b1d28 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:55:34 +1000 Subject: [PATCH 16/81] Change event memory model --- components/livekit/core/engine.c | 119 +++++++++++++++++-------------- components/livekit/core/engine.h | 2 +- 2 files changed, 65 insertions(+), 56 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index cd4531e..3c3c009 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -39,8 +39,8 @@ typedef struct { union { /// Detail for `EV_CMD_CONNECT`. struct { - const char *server_url; - const char *token; + char *server_url; + char *token; } cmd_connect; /// Detail for `EV_SIG_STATE`. @@ -99,8 +99,8 @@ static void event_free(engine_event_t *ev) if (ev == NULL) return; switch (ev->type) { case EV_CMD_CONNECT: - free((char *)ev->detail.cmd_connect.server_url); - free((char *)ev->detail.cmd_connect.token); + free(ev->detail.cmd_connect.server_url); + free(ev->detail.cmd_connect.token); break; default: break; } @@ -539,7 +539,7 @@ static bool establish_peer_connections(engine_t *eng) // MARK: - Connection state machine -static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); +static bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); static void flush_event_queue(engine_t *eng); static void engine_task(void *arg) @@ -555,8 +555,16 @@ static void engine_task(void *arg) ESP_LOGI(TAG, "Event: %d", ev.type); engine_state_t state = eng->state; - handle_state(eng, &ev, state); + // Invoke the handler for the current state, passing the event that woke up the + // state machine. The handler returns whether or not the event was handled; if + // the handler handles the event, it takes ownership of any dynamic fields in the + // event's detail and is responsible for storing or freeing them. + if (!handle_state(eng, &ev, state)) + event_free(&ev); + + // If the state changed, invoke the exit handler for the old state, + // the enter handler for the new state, and notify. if (eng->state != state) { ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); @@ -566,14 +574,14 @@ static void engine_task(void *arg) handle_state(eng, &(engine_event_t){ .type = _EV_STATE_ENTER }, eng->state); assert(eng->state == state); + // TODO: Notify of state change } - event_free(&ev); } flush_event_queue(eng); vTaskDelete(NULL); } -static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) +static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -582,46 +590,44 @@ static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) signal_close(eng->signal_handle); destroy_peer_connections(eng); eng->retry_count = 0; - break; + return true; case EV_CMD_CONNECT: - const char *server_url = ev->detail.cmd_connect.server_url; - const char *token = ev->detail.cmd_connect.token; - if (eng->server_url != NULL) free(eng->server_url); if (eng->token != NULL) free(eng->token); - eng->server_url = strdup(server_url); - eng->token = strdup(token); + eng->server_url = ev->detail.cmd_connect.server_url; // Take ownership + eng->token = ev->detail.cmd_connect.token; eng->state = ENGINE_STATE_CONNECTING; - break; - default: break; + return true; + default: + return false; } } -static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) +static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: signal_connect(eng->signal_handle, eng->server_url, eng->token); - break; + return true; case EV_CMD_CLOSE: // TODO: Send leave request eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connecting, ignoring connect command"); - break; + return false; case EV_SIG_STATE: // TODO: Check error code (4xx should go to disconnected) if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; + return true; case EV_SIG_LEAVE: ESP_LOGI(TAG, "Server sent leave before fully connected"); eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; case EV_SIG_JOIN: eng->is_subscriber_primary = ev->detail.sig_join.subscriber_primary; eng->force_relay = ev->detail.sig_join.force_relay; @@ -629,75 +635,77 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) if (!establish_peer_connections(eng)) { ESP_LOGE(TAG, "Failed to establish peer connections"); eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; } - break; + return true; case EV_PEER_PUB_STATE: connection_state_t pub_state = ev->detail.peer_state.state; if (!eng->is_subscriber_primary && pub_state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; } else if (pub_state == CONNECTION_STATE_FAILED || pub_state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; + return true; case EV_PEER_SUB_STATE: connection_state_t sub_state = ev->detail.peer_state.state; if (eng->is_subscriber_primary && sub_state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; } else if (sub_state == CONNECTION_STATE_FAILED || sub_state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; - default: break; + return true; + default: + return false; } } -static void handle_state_connected(engine_t *eng, const engine_event_t *ev) +static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: eng->retry_count = 0; publish_tracks(eng); - break; + return true; case EV_CMD_CLOSE: // TODO: Send leave request eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); - break; + return false; case EV_SIG_STATE: if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; + return true; case EV_SIG_LEAVE: ESP_LOGI(TAG, "Server initiated disconnect"); // TODO: Handle leave action eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; case EV_PEER_PUB_STATE: connection_state_t pub_state = ev->detail.peer_state.state; if (pub_state == CONNECTION_STATE_FAILED || pub_state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; + return true; case EV_PEER_SUB_STATE: connection_state_t sub_state = ev->detail.peer_state.state; if (sub_state == CONNECTION_STATE_FAILED || sub_state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_RECONNECTING; + eng->state = ENGINE_STATE_BACKOFF; } - break; - default: break; + return true; + default: + return false; } } -static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) +static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -708,7 +716,7 @@ static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); - break; + return true; } // TODO: Exponential backoff uint32_t backoff_ms = 1000; @@ -718,29 +726,30 @@ static void handle_state_reconnecting(engine_t *eng, const engine_event_t *ev) xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); xTimerStart(eng->timer, 0); - break; + return true; case EV_MAX_RETRIES_REACHED: eng->state = ENGINE_STATE_DISCONNECTED; - break; + return true; case EV_TIMER_EXP: eng->retry_count++; eng->state = ENGINE_STATE_CONNECTING; - break; + return true; case _EV_STATE_EXIT: xTimerStop(eng->timer, portMAX_DELAY); - break; - default: break; + return true; + default: + return false; } } -static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) +static inline bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) { switch (state) { - case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng, ev); break; - case ENGINE_STATE_CONNECTING: handle_state_connecting(eng, ev); break; - case ENGINE_STATE_CONNECTED: handle_state_connected(eng, ev); break; - case ENGINE_STATE_RECONNECTING: handle_state_reconnecting(eng, ev); break; - default: break; + case ENGINE_STATE_DISCONNECTED: return handle_state_disconnected(eng, ev); + case ENGINE_STATE_CONNECTING: return handle_state_connecting(eng, ev); + case ENGINE_STATE_CONNECTED: return handle_state_connected(eng, ev); + case ENGINE_STATE_BACKOFF: return handle_state_backoff(eng, ev); + default: esp_system_abort("Unknown engine state"); } } diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 8d9f60b..99e6e31 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -22,7 +22,7 @@ typedef enum { ENGINE_STATE_DISCONNECTED, ENGINE_STATE_CONNECTING, ENGINE_STATE_CONNECTED, - ENGINE_STATE_RECONNECTING + ENGINE_STATE_BACKOFF } engine_state_t; typedef enum { From 043efdec15fbffd921d40943bbdbbf3a438b7889 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:33:49 +1000 Subject: [PATCH 17/81] Store local participant ID --- components/livekit/core/engine.c | 13 ++++++++++++- components/livekit/core/protocol.h | 5 ++++- components/livekit/protocol/livekit_models.pb.h | 8 ++++---- .../protocol/protobufs/livekit_models.options | 2 +- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 3c3c009..9b3f223 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -52,7 +52,7 @@ typedef struct { struct { bool subscriber_primary; bool force_relay; - char local_participant_sid[32]; + livekit_pb_sid_t local_participant_sid; } sig_join; /// Detail for `EV_SIG_LEAVE`. @@ -85,6 +85,7 @@ typedef struct { bool force_relay; char* server_url; char* token; + livekit_pb_sid_t local_participant_sid; TaskHandle_t task_handle; QueueHandle_t event_queue; @@ -334,6 +335,11 @@ static void on_signal_join(livekit_pb_join_response_t *join_res, void *ctx) join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED } }; + strncpy( + ev.detail.sig_join.local_participant_sid, + join_res->participant.sid, + sizeof(ev.detail.sig_join.local_participant_sid) + ); xQueueSendToFront(eng->event_queue, &ev, 0); } @@ -631,6 +637,11 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) case EV_SIG_JOIN: eng->is_subscriber_primary = ev->detail.sig_join.subscriber_primary; eng->force_relay = ev->detail.sig_join.force_relay; + strncpy( + eng->local_participant_sid, + ev->detail.sig_join.local_participant_sid, + sizeof(eng->local_participant_sid) + ); if (!establish_peer_connections(eng)) { ESP_LOGE(TAG, "Failed to establish peer connections"); diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 7ef6933..b008fed 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -7,4 +7,7 @@ #include "livekit_rtc.pb.h" #include "livekit_models.pb.h" #include "livekit_metrics.pb.h" -#include "timestamp.pb.h" \ No newline at end of file +#include "timestamp.pb.h" + +/// Server identifier (SID) type. +typedef char livekit_pb_sid_t[16]; \ No newline at end of file diff --git a/components/livekit/protocol/livekit_models.pb.h b/components/livekit/protocol/livekit_models.pb.h index 287353e..b56cc64 100644 --- a/components/livekit/protocol/livekit_models.pb.h +++ b/components/livekit/protocol/livekit_models.pb.h @@ -244,7 +244,7 @@ typedef struct livekit_pb_participant_permission { } livekit_pb_participant_permission_t; typedef struct livekit_pb_participant_info { - char *sid; + char sid[16]; char *identity; livekit_pb_participant_info_state_t state; pb_size_t tracks_count; @@ -767,7 +767,7 @@ extern "C" { #define LIVEKIT_PB_CODEC_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_PLAYOUT_DELAY_INIT_DEFAULT {0, 0, 0} #define LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT {0, 0, 0} -#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_DEFAULT {NULL, NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT, NULL} +#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_DEFAULT {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT, NULL} #define LIVEKIT_PB_ENCRYPTION_INIT_DEFAULT {0} #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_DEFAULT {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} @@ -811,7 +811,7 @@ extern "C" { #define LIVEKIT_PB_CODEC_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_PLAYOUT_DELAY_INIT_ZERO {0, 0, 0} #define LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO {0, 0, 0} -#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_ZERO {NULL, NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO, NULL} +#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_ZERO {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO, NULL} #define LIVEKIT_PB_ENCRYPTION_INIT_ZERO {0} #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_ZERO {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} @@ -1109,7 +1109,7 @@ X(a, STATIC, SINGULAR, BOOL, can_publish_data, 3) #define LIVEKIT_PB_PARTICIPANT_PERMISSION_DEFAULT NULL #define LIVEKIT_PB_PARTICIPANT_INFO_FIELDLIST(X, a) \ -X(a, POINTER, SINGULAR, STRING, sid, 1) \ +X(a, STATIC, SINGULAR, STRING, sid, 1) \ X(a, POINTER, SINGULAR, STRING, identity, 2) \ X(a, STATIC, SINGULAR, UENUM, state, 3) \ X(a, POINTER, REPEATED, MESSAGE, tracks, 4) \ diff --git a/components/livekit/protocol/protobufs/livekit_models.options b/components/livekit/protocol/protobufs/livekit_models.options index c93b551..f52dcc2 100644 --- a/components/livekit/protocol/protobufs/livekit_models.options +++ b/components/livekit/protocol/protobufs/livekit_models.options @@ -2,7 +2,7 @@ livekit_pb.ClientConfiguration.video type:FT_IGNORE livekit_pb.ClientConfiguration.screen type:FT_IGNORE livekit_pb.ClientConfiguration.disabled_codecs type:FT_IGNORE -livekit_pb.ParticipantInfo.sid type:FT_POINTER +livekit_pb.ParticipantInfo.sid max_length:15 livekit_pb.ParticipantInfo.identity type:FT_POINTER livekit_pb.ParticipantInfo.tracks type:FT_POINTER livekit_pb.ParticipantInfo.metadata type:FT_POINTER From 5f88fb610eda8985f9725299c7326316ba6de202 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:40:44 +1000 Subject: [PATCH 18/81] Consolidate event detail cases --- components/livekit/core/engine.c | 51 +++++++++++++------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 9b3f223..199d8dc 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -43,11 +43,6 @@ typedef struct { char *token; } cmd_connect; - /// Detail for `EV_SIG_STATE`. - struct { - connection_state_t state; - } sig_state; - /// Detail for `EV_SIG_JOIN`. struct { bool subscriber_primary; @@ -61,10 +56,8 @@ typedef struct { livekit_pb_leave_request_action_t action; } sig_leave; - /// Detail for `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. - struct { - connection_state_t state; - } peer_state; + /// Detail for `EV_SIG_STATE`, `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. + connection_state_t state; } detail; } engine_event_t; @@ -319,7 +312,7 @@ static void on_signal_state_changed(connection_state_t state, void *ctx) engine_t *eng = (engine_t *)ctx; engine_event_t ev = { .type = EV_SIG_STATE, - .detail.sig_state = { .state = state } + .detail.state = state }; xQueueSendToFront(eng->event_queue, &ev, 0); } @@ -416,7 +409,7 @@ static void on_peer_pub_state_changed(connection_state_t state, void *ctx) engine_t *eng = (engine_t *)ctx; engine_event_t ev = { .type = EV_PEER_PUB_STATE, - .detail.peer_state = { .state = state } + .detail.state = state }; xQueueSendToFront(eng->event_queue, &ev, 0); } @@ -434,7 +427,7 @@ static void on_peer_sub_state_changed(connection_state_t state, void *ctx) engine_t *eng = (engine_t *)ctx; engine_event_t ev = { .type = EV_PEER_SUB_STATE, - .detail.peer_state = { .state = state } + .detail.state = state }; xQueueSendToFront(eng->event_queue, &ev, 0); } @@ -625,8 +618,8 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) return false; case EV_SIG_STATE: // TODO: Check error code (4xx should go to disconnected) - if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || - ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; @@ -650,20 +643,20 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } return true; case EV_PEER_PUB_STATE: - connection_state_t pub_state = ev->detail.peer_state.state; - if (!eng->is_subscriber_primary && pub_state == CONNECTION_STATE_CONNECTED) { + if (!eng->is_subscriber_primary && + ev->detail.state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; - } else if (pub_state == CONNECTION_STATE_FAILED || - pub_state == CONNECTION_STATE_DISCONNECTED) { + } else if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; case EV_PEER_SUB_STATE: - connection_state_t sub_state = ev->detail.peer_state.state; - if (eng->is_subscriber_primary && sub_state == CONNECTION_STATE_CONNECTED) { + if (eng->is_subscriber_primary && + ev->detail.state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; - } else if (sub_state == CONNECTION_STATE_FAILED || - sub_state == CONNECTION_STATE_DISCONNECTED) { + } else if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; @@ -687,8 +680,8 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); return false; case EV_SIG_STATE: - if (ev->detail.sig_state.state == CONNECTION_STATE_FAILED || - ev->detail.sig_state.state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; @@ -698,16 +691,14 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_DISCONNECTED; return true; case EV_PEER_PUB_STATE: - connection_state_t pub_state = ev->detail.peer_state.state; - if (pub_state == CONNECTION_STATE_FAILED || - pub_state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; case EV_PEER_SUB_STATE: - connection_state_t sub_state = ev->detail.peer_state.state; - if (sub_state == CONNECTION_STATE_FAILED || - sub_state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED || + ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; From 7a1a3094693d20e617412126275c4daee62c479f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:46:17 +1000 Subject: [PATCH 19/81] Cleanup session state --- components/livekit/core/engine.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 199d8dc..5b9e993 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -588,7 +588,12 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) media_stream_end(eng); signal_close(eng->signal_handle); destroy_peer_connections(eng); + + eng->is_subscriber_primary = false; + eng->force_relay = false; + eng->local_participant_sid[0] = '\0'; eng->retry_count = 0; + return true; case EV_CMD_CONNECT: if (eng->server_url != NULL) free(eng->server_url); From 1832a9ce4518273f12956039706db691f70e1b39 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:23:06 +1000 Subject: [PATCH 20/81] Engine refactor - Handle all signal responses in state machine - Helper functions for protocol messages --- components/livekit/core/engine.c | 228 +++++++++++++++------------- components/livekit/core/protocol.c | 57 +++++++ components/livekit/core/protocol.h | 29 +++- components/livekit/core/signaling.c | 131 ++++------------ components/livekit/core/signaling.h | 17 ++- 5 files changed, 245 insertions(+), 217 deletions(-) create mode 100644 components/livekit/core/protocol.c diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 5b9e993..96a24d5 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -22,8 +22,7 @@ typedef enum { EV_CMD_CONNECT, /// User-initiated connect. EV_CMD_CLOSE, /// User-initiated disconnect. EV_SIG_STATE, /// Signal state changed. - EV_SIG_JOIN, /// Join response received. - EV_SIG_LEAVE, /// Leave request received. + EV_SIG_RES, /// Signal response received. EV_PEER_PUB_STATE, /// Publisher peer state changed. EV_PEER_SUB_STATE, /// Subscriber peer state changed. EV_TIMER_EXP, /// Timer expired. @@ -43,18 +42,8 @@ typedef struct { char *token; } cmd_connect; - /// Detail for `EV_SIG_JOIN`. - struct { - bool subscriber_primary; - bool force_relay; - livekit_pb_sid_t local_participant_sid; - } sig_join; - - /// Detail for `EV_SIG_LEAVE`. - struct { - livekit_pb_disconnect_reason_t reason; - livekit_pb_leave_request_action_t action; - } sig_leave; + /// Detail for `EV_SIG_RES`. + livekit_pb_signal_response_t res; /// Detail for `EV_SIG_STATE`, `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. connection_state_t state; @@ -96,6 +85,9 @@ static void event_free(engine_event_t *ev) free(ev->detail.cmd_connect.server_url); free(ev->detail.cmd_connect.token); break; + case EV_SIG_RES: + protocol_signal_res_free(&ev->detail.res); + break; default: break; } } @@ -317,74 +309,19 @@ static void on_signal_state_changed(connection_state_t state, void *ctx) xQueueSendToFront(eng->event_queue, &ev, 0); } -static void on_signal_join(livekit_pb_join_response_t *join_res, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - engine_event_t ev = { - .type = EV_SIG_JOIN, - .detail.sig_join = { - .subscriber_primary = join_res->subscriber_primary, - .force_relay = join_res->has_client_configuration && - join_res->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED - } - }; - strncpy( - ev.detail.sig_join.local_participant_sid, - join_res->participant.sid, - sizeof(ev.detail.sig_join.local_participant_sid) - ); - xQueueSendToFront(eng->event_queue, &ev, 0); -} - -static void on_signal_leave(livekit_pb_disconnect_reason_t reason, livekit_pb_leave_request_action_t action, void *ctx) +static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) { engine_t *eng = (engine_t *)ctx; engine_event_t ev = { - .type = EV_SIG_LEAVE, - .detail.sig_leave = { .reason = reason, .action = action } + .type = EV_SIG_RES, + .detail.res = *res }; - xQueueSendToFront(eng->event_queue, &ev, 0); -} - -static void on_signal_answer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - peer_handle_sdp(eng->pub_peer_handle, sdp); -} - -static void on_signal_offer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - peer_handle_sdp(eng->sub_peer_handle, sdp); -} - -static void on_signal_trickle(const char *ice_candidate, livekit_pb_signal_target_t target, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - peer_handle_t target_peer = target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? - eng->sub_peer_handle : eng->pub_peer_handle; - peer_handle_ice_candidate(target_peer, ice_candidate); -} - -static void on_signal_room_update(const livekit_pb_room_t* info, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - if (eng->options.on_room_info) { - eng->options.on_room_info(info, eng->options.ctx); + // Returning true takes ownership of the response; it will be freed later when the + // queue is processed or flushed. + if (res->which_message == LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG) { + return xQueueSendToFront(eng->event_queue, &ev, 0) == pdPASS; } -} - -static void on_signal_participant_update(const livekit_pb_participant_info_t* info, void *ctx) -{ - // engine_t *eng = (engine_t *)ctx; - // TODO: Subscribe - // bool is_local = strncmp(info->sid, eng->local_participant_sid, sizeof(eng->local_participant_sid)) == 0; - // if (eng->options.on_participant_info) { - // eng->options.on_participant_info(info, is_local, eng->options.ctx); - //} - - // if (is_local) return; - // subscribe_tracks(eng, info->tracks, info->tracks_count); + return xQueueSend(eng->event_queue, &ev, 0) == pdPASS; } // MARK: - Common peer event handlers @@ -621,6 +558,56 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connecting, ignoring connect command"); return false; + case EV_SIG_RES: + livekit_pb_signal_response_t *res = &ev->detail.res; + switch (res->which_message) { + case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: + ESP_LOGI(TAG, "Server sent leave before fully connected"); + eng->state = ENGINE_STATE_DISCONNECTED; + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: + livekit_pb_join_response_t *join = &res->message.join; + // Store connection settings + eng->is_subscriber_primary = join->subscriber_primary; + if (join->has_client_configuration) { + eng->force_relay = join->client_configuration.force_relay + == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; + } + // Store local participant SID + strncpy( + eng->local_participant_sid, + join->participant.sid, + sizeof(eng->local_participant_sid) + ); + if (!establish_peer_connections(eng)) { + ESP_LOGE(TAG, "Failed to establish peer connections"); + eng->state = ENGINE_STATE_DISCONNECTED; + return true; + } + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: + livekit_pb_session_description_t *answer = &res->message.answer; + peer_handle_sdp(eng->pub_peer_handle, answer->sdp); + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: + livekit_pb_session_description_t *offer = &res->message.offer; + peer_handle_sdp(eng->sub_peer_handle, offer->sdp); + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: + livekit_pb_trickle_request_t *trickle = &res->message.trickle; + char* candidate = NULL; + if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { + return true; + } + peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? + eng->pub_peer_handle : eng->sub_peer_handle; + peer_handle_ice_candidate(target_peer, candidate); + free(candidate); + return true; + default: + return true; + } + return true; case EV_SIG_STATE: // TODO: Check error code (4xx should go to disconnected) if (ev->detail.state == CONNECTION_STATE_FAILED || @@ -628,25 +615,6 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_BACKOFF; } return true; - case EV_SIG_LEAVE: - ESP_LOGI(TAG, "Server sent leave before fully connected"); - eng->state = ENGINE_STATE_DISCONNECTED; - return true; - case EV_SIG_JOIN: - eng->is_subscriber_primary = ev->detail.sig_join.subscriber_primary; - eng->force_relay = ev->detail.sig_join.force_relay; - strncpy( - eng->local_participant_sid, - ev->detail.sig_join.local_participant_sid, - sizeof(eng->local_participant_sid) - ); - - if (!establish_peer_connections(eng)) { - ESP_LOGE(TAG, "Failed to establish peer connections"); - eng->state = ENGINE_STATE_DISCONNECTED; - return true; - } - return true; case EV_PEER_PUB_STATE: if (!eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { @@ -684,17 +652,67 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); return false; + case EV_SIG_RES: + livekit_pb_signal_response_t *res = &ev->detail.res; + switch (ev->detail.res.which_message) { + case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: + ESP_LOGI(TAG, "Server initiated disconnect"); + // TODO: Handle leave action + eng->state = ENGINE_STATE_DISCONNECTED; + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: + livekit_pb_room_update_t *room_update = &res->message.room_update; + if (eng->options.on_room_info && room_update->has_room) { + eng->options.on_room_info(&room_update->room, eng->options.ctx); + } + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: + livekit_pb_participant_update_t *update = &res->message.update; + if (!eng->options.on_participant_info) { + return true; + } + bool found_local = false; + for (pb_size_t i = 0; i < update->participants_count; i++) { + livekit_pb_participant_info_t *participant = &update->participants[i]; + bool is_local = !found_local && strncmp( + participant->sid, + eng->local_participant_sid, + sizeof(eng->local_participant_sid) + ) == 0; + if (is_local) found_local = true; + eng->options.on_participant_info(participant, is_local, eng->options.ctx); + } + return true; + // TODO: Only handle if needed + case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: + livekit_pb_session_description_t *answer = &res->message.answer; + peer_handle_sdp(eng->pub_peer_handle, answer->sdp); + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: + livekit_pb_session_description_t *offer = &res->message.offer; + peer_handle_sdp(eng->sub_peer_handle, offer->sdp); + return true; + case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: + livekit_pb_trickle_request_t *trickle = &res->message.trickle; + char* candidate = NULL; + if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { + return true; + } + peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? + eng->pub_peer_handle : eng->sub_peer_handle; + peer_handle_ice_candidate(target_peer, candidate); + free(candidate); + return true; + default: + return true; + } + return true; case EV_SIG_STATE: if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } return true; - case EV_SIG_LEAVE: - ESP_LOGI(TAG, "Server initiated disconnect"); - // TODO: Handle leave action - eng->state = ENGINE_STATE_DISCONNECTED; - return true; case EV_PEER_PUB_STATE: if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { @@ -802,13 +820,7 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) signal_options_t signal_options = { .ctx = eng, .on_state_changed = on_signal_state_changed, - .on_join = on_signal_join, - .on_leave = on_signal_leave, - .on_answer = on_signal_answer, - .on_offer = on_signal_offer, - .on_trickle = on_signal_trickle, - .on_room_update = on_signal_room_update, - .on_participant_update = on_signal_participant_update + .on_res = on_signal_res, }; if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { free(eng->event_queue); diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c new file mode 100644 index 0000000..b058a17 --- /dev/null +++ b/components/livekit/core/protocol.c @@ -0,0 +1,57 @@ +#include "esp_log.h" +#include "cJSON.h" +#include "protocol.h" + +static const char *TAG = "livekit_protocol"; + +bool protocol_signal_res_decode(const char *buf, size_t len, livekit_pb_signal_response_t* out) +{ + pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); + if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, out)) { + ESP_LOGE(TAG, "Failed to decode signal res: %s", stream.errmsg); + return false; + } + return true; +} + +void protocol_signal_res_free(livekit_pb_signal_response_t *res) +{ + pb_release(LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, res); +} + +bool protocol_signal_trickle_get_candidate(const livekit_pb_trickle_request_t *trickle, char **candidate_out) +{ + if (trickle == NULL || candidate_out == NULL) { + return false; + } + if (trickle->candidate_init == NULL) { + ESP_LOGE(TAG, "candidate_init is NULL"); + return false; + } + + bool ret = false; + cJSON *candidate_init = NULL; + do { + candidate_init = cJSON_Parse(trickle->candidate_init); + if (candidate_init == NULL) { + const char *error_ptr = cJSON_GetErrorPtr(); + if (error_ptr != NULL) { + ESP_LOGE(TAG, "Failed to parse candidate_init: %s", error_ptr); + } + break; + } + cJSON *candidate = cJSON_GetObjectItemCaseSensitive(candidate_init, "candidate"); + if (!cJSON_IsString(candidate) || (candidate->valuestring == NULL)) { + ESP_LOGE(TAG, "Missing candidate key in candidate_init"); + break; + } + *candidate_out = strdup(candidate->valuestring); + if (*candidate_out == NULL) { + break; + } + ret = true; + } while(0); + + cJSON_Delete(candidate_init); + return ret; +} \ No newline at end of file diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index b008fed..9402566 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -9,5 +9,32 @@ #include "livekit_metrics.pb.h" #include "timestamp.pb.h" +#ifdef __cplusplus +extern "C" { +#endif + /// Server identifier (SID) type. -typedef char livekit_pb_sid_t[16]; \ No newline at end of file +typedef char livekit_pb_sid_t[16]; + +/// Decodes a signal response. +bool protocol_signal_res_decode(const char *buf, size_t len, livekit_pb_signal_response_t* out); + +/// Frees a signal response. +/// +/// Always use this to discard signal responses, even for messages types that do not have +/// dynamic fields as they may be added in the future. +/// +void protocol_signal_res_free(livekit_pb_signal_response_t *res); + +/// Extract ICE candidate string from a trickle request. +/// +/// The caller is responsible for freeing the candidate string if it is not NULL. +/// +bool protocol_signal_trickle_get_candidate( + const livekit_pb_trickle_request_t *trickle, + char **candidate_out +); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 8d38885..0d11c85 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -88,113 +88,32 @@ static void send_ping(void *arg) send_request(sg, &req); } -static void handle_res(signal_t *sg, livekit_pb_signal_response_t *res) +/// Processes responses before forwarding them to the receiver. +static inline bool res_middleware(signal_t *sg, livekit_pb_signal_response_t *res) { + if (res->which_message != LIVEKIT_PB_SIGNAL_RESPONSE_PONG_RESP_TAG && + res->which_message != LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG) { + return true; + } + bool should_forward = false; switch (res->which_message) { case LIVEKIT_PB_SIGNAL_RESPONSE_PONG_RESP_TAG: livekit_pb_pong_t *pong = &res->message.pong_resp; sg->rtt = get_unix_time_ms() - pong->last_ping_timestamp; // TODO: Reset ping timeout - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_REFRESH_TOKEN_TAG: - // TODO: Handle refresh token + should_forward = false; break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: - livekit_pb_join_response_t *join_res = &res->message.join; - ESP_LOGI(TAG, "Join: subscriber_primary=%d", join_res->subscriber_primary); - - sg->ping_interval_ms = join_res->ping_interval * 1000; - sg->ping_timeout_ms = join_res->ping_timeout * 1000; + livekit_pb_join_response_t *join = &res->message.join; + sg->ping_interval_ms = join->ping_interval * 1000; + sg->ping_timeout_ms = join->ping_timeout * 1000; esp_timer_start_periodic(sg->ping_timer, sg->ping_interval_ms * 1000); - - if (sg->options.on_join != NULL) { - sg->options.on_join(join_res, sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: - livekit_pb_leave_request_t *leave_res = &res->message.leave; - ESP_LOGI(TAG, "Leave: reason=%d, action=%d", leave_res->reason, leave_res->action); - esp_timer_stop(sg->ping_timer); - if (sg->options.on_leave != NULL) { - sg->options.on_leave(leave_res->reason, leave_res->action, sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: - livekit_pb_room_update_t *room_update = &res->message.room_update; - if (!room_update->has_room) break; - if (sg->options.on_room_update != NULL) { - sg->options.on_room_update(&room_update->room, sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: - livekit_pb_participant_update_t *participant_update = &res->message.update; - if (sg->options.on_participant_update == NULL) break; - for (int i = 0; i < participant_update->participants_count; i++) { - sg->options.on_participant_update(&participant_update->participants[i], sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: - livekit_pb_session_description_t *offer = &res->message.offer; - ESP_LOGI(TAG, "Offer: id=%" PRIu32 "\n%s", offer->id, offer->sdp); - if (sg->options.on_offer != NULL) { - sg->options.on_offer(offer->sdp, sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: - livekit_pb_session_description_t *answer = &res->message.answer; - ESP_LOGI(TAG, "Answer: id=%" PRIu32 "\n%s", answer->id, answer->sdp); - if (sg->options.on_answer != NULL) { - sg->options.on_answer(answer->sdp, sg->options.ctx); - } - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: - livekit_pb_trickle_request_t *trickle = &res->message.trickle; - if (trickle->candidate_init == NULL) { - ESP_LOGE(TAG, "Trickle candidate_init is NULL"); - break; - } - cJSON *candidate_init = NULL; - do { - candidate_init = cJSON_Parse(trickle->candidate_init); - if (candidate_init == NULL) { - const char *error_ptr = cJSON_GetErrorPtr(); - if (error_ptr != NULL) { - ESP_LOGE(TAG, "Failed to parse candidate_init: %s", error_ptr); - } - break; - } - cJSON *candidate = cJSON_GetObjectItemCaseSensitive(candidate_init, "candidate"); - if (!cJSON_IsString(candidate) || (candidate->valuestring == NULL)) { - ESP_LOGE(TAG, "Missing candidate key in candidate_init"); - break; - } - ESP_LOGI(TAG, "Trickle: target=%d, final=%d\n%s", - trickle->target, - trickle->final, - candidate->valuestring - ); - if (sg->options.on_trickle != NULL) { - sg->options.on_trickle(candidate->valuestring, trickle->target, sg->options.ctx); - } - } while (0); - cJSON_Delete(candidate_init); + should_forward = true; break; default: - break; + should_forward = false; } - pb_release(LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, res); -} - -static void on_data(signal_t *sg, const char *data, size_t len) -{ - ESP_LOGD(TAG, "Incoming res: %d byte(s)", len); - livekit_pb_signal_response_t res = {}; - pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)data, len); - if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, &res)) { - ESP_LOGE(TAG, "Failed to decode res: %s", stream.errmsg); - return; - } - handle_res(sg, &res); + return should_forward; } static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void *event_data) @@ -210,9 +129,6 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void break; case WEBSOCKET_EVENT_DISCONNECTED: ESP_LOGD(TAG, "Signaling disconnected"); - - // In the normal case, this timer will be stopped when the leave message is received. - // However, if the connection is lost, we need to stop the timer manually. esp_timer_stop(sg->ping_timer); log_error_if_nonzero("HTTP status code", data->error_handle.esp_ws_handshake_status_code); @@ -225,11 +141,23 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void break; case WEBSOCKET_EVENT_DATA: if (data->op_code != WS_TRANSPORT_OPCODES_BINARY) { - ESP_LOGD(TAG, "Message: opcode=%d, len=%d", data->op_code, data->data_len); + ESP_LOGW(TAG, "Received non-binary message"); break; } if (data->data_len < 1) break; - on_data(sg, data->data_ptr, data->data_len); + livekit_pb_signal_response_t res; + if (!protocol_signal_res_decode(data->data_ptr, data->data_len, &res)) { + break; + } + if (!res_middleware(sg, &res)) { + // Don't forward. + protocol_signal_res_free(&res); + break; + } + if (!sg->options.on_res(&res, sg->options.ctx)) { + // Ownership was not taken. + protocol_signal_res_free(&res); + } break; case WEBSOCKET_EVENT_ERROR: ESP_LOGE(TAG, "Failed to connect to server"); @@ -253,7 +181,8 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) return SIGNAL_ERR_INVALID_ARG; } - if (options->on_state_changed == NULL) { + if (options->on_state_changed == NULL || + options->on_res == NULL) { ESP_LOGE(TAG, "Missing required event handlers"); return SIGNAL_ERR_INVALID_ARG; } diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index b977d47..af16ce7 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -23,14 +23,17 @@ typedef enum { typedef struct { void* ctx; + + /// Invoked when the connection state changes. void (*on_state_changed)(connection_state_t state, void *ctx); - void (*on_join)(livekit_pb_join_response_t *join_res, void *ctx); - void (*on_leave)(livekit_pb_disconnect_reason_t reason, livekit_pb_leave_request_action_t action, void *ctx); - void (*on_room_update)(const livekit_pb_room_t* info, void *ctx); - void (*on_participant_update)(const livekit_pb_participant_info_t* info, void *ctx); - void (*on_answer)(const char *sdp, void *ctx); - void (*on_offer)(const char *sdp, void *ctx); - void (*on_trickle)(const char *ice_candidate, livekit_pb_signal_target_t target, void *ctx); + + /// Invoked when a signal response is received. + /// + /// The receiver returns true to take ownership of the response. If + /// ownership is not taken (false), the response will be freed with + /// `protocol_signal_res_free` internally. + /// + bool (*on_res)(livekit_pb_signal_response_t *res, void *ctx); } signal_options_t; signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options); From a2eae8f17e010bc1650959e0d4e4c3dea47f3006 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:37:46 +1000 Subject: [PATCH 21/81] Simplify state machine memory model Always discard events --- components/livekit/core/engine.c | 125 ++++++++++++++++--------------- 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 96a24d5..f4366d5 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -82,8 +82,10 @@ static void event_free(engine_event_t *ev) if (ev == NULL) return; switch (ev->type) { case EV_CMD_CONNECT: - free(ev->detail.cmd_connect.server_url); - free(ev->detail.cmd_connect.token); + if (ev->detail.cmd_connect.server_url != NULL) + free(ev->detail.cmd_connect.server_url); + if (ev->detail.cmd_connect.token != NULL) + free(ev->detail.cmd_connect.token); break; case EV_SIG_RES: protocol_signal_res_free(&ev->detail.res); @@ -475,7 +477,7 @@ static bool establish_peer_connections(engine_t *eng) // MARK: - Connection state machine -static bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); +static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); static void flush_event_queue(engine_t *eng); static void engine_task(void *arg) @@ -493,11 +495,9 @@ static void engine_task(void *arg) engine_state_t state = eng->state; // Invoke the handler for the current state, passing the event that woke up the - // state machine. The handler returns whether or not the event was handled; if - // the handler handles the event, it takes ownership of any dynamic fields in the - // event's detail and is responsible for storing or freeing them. - if (!handle_state(eng, &ev, state)) - event_free(&ev); + // state machine, then free the event once the handler has completed. + handle_state(eng, &ev, state); + event_free(&ev); // If the state changed, invoke the exit handler for the old state, // the enter handler for the new state, and notify. @@ -513,11 +513,13 @@ static void engine_task(void *arg) // TODO: Notify of state change } } + + // Discard any remaining events in the queue before exiting. flush_event_queue(eng); vTaskDelete(NULL); } -static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) +static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -530,41 +532,40 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) eng->force_relay = false; eng->local_participant_sid[0] = '\0'; eng->retry_count = 0; - - return true; + break; case EV_CMD_CONNECT: if (eng->server_url != NULL) free(eng->server_url); if (eng->token != NULL) free(eng->token); - eng->server_url = ev->detail.cmd_connect.server_url; // Take ownership - eng->token = ev->detail.cmd_connect.token; + eng->server_url = strdup(ev->detail.cmd_connect.server_url); + eng->token = strdup(ev->detail.cmd_connect.token); eng->state = ENGINE_STATE_CONNECTING; - return true; + break; default: - return false; + break; } } -static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) +static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: signal_connect(eng->signal_handle, eng->server_url, eng->token); - return true; + break; case EV_CMD_CLOSE: // TODO: Send leave request eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connecting, ignoring connect command"); - return false; + break; case EV_SIG_RES: livekit_pb_signal_response_t *res = &ev->detail.res; switch (res->which_message) { case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: ESP_LOGI(TAG, "Server sent leave before fully connected"); eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: livekit_pb_join_response_t *join = &res->message.join; // Store connection settings @@ -582,39 +583,39 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) if (!establish_peer_connections(eng)) { ESP_LOGE(TAG, "Failed to establish peer connections"); eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; } - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: livekit_pb_session_description_t *answer = &res->message.answer; peer_handle_sdp(eng->pub_peer_handle, answer->sdp); - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: livekit_pb_session_description_t *offer = &res->message.offer; peer_handle_sdp(eng->sub_peer_handle, offer->sdp); - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: livekit_pb_trickle_request_t *trickle = &res->message.trickle; char* candidate = NULL; if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { - return true; + break; } peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? eng->pub_peer_handle : eng->sub_peer_handle; peer_handle_ice_candidate(target_peer, candidate); free(candidate); - return true; + break; default: - return true; + break; } - return true; + break; case EV_SIG_STATE: // TODO: Check error code (4xx should go to disconnected) if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; case EV_PEER_PUB_STATE: if (!eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { @@ -623,7 +624,7 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; case EV_PEER_SUB_STATE: if (eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { @@ -632,26 +633,26 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; default: - return false; + break; } } -static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) +static void handle_state_connected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: eng->retry_count = 0; publish_tracks(eng); - return true; + break; case EV_CMD_CLOSE: // TODO: Send leave request eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); - return false; + break; case EV_SIG_RES: livekit_pb_signal_response_t *res = &ev->detail.res; switch (ev->detail.res.which_message) { @@ -659,17 +660,17 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) ESP_LOGI(TAG, "Server initiated disconnect"); // TODO: Handle leave action eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: livekit_pb_room_update_t *room_update = &res->message.room_update; if (eng->options.on_room_info && room_update->has_room) { eng->options.on_room_info(&room_update->room, eng->options.ctx); } - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: livekit_pb_participant_update_t *update = &res->message.update; if (!eng->options.on_participant_info) { - return true; + break; } bool found_local = false; for (pb_size_t i = 0; i < update->participants_count; i++) { @@ -682,55 +683,55 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) if (is_local) found_local = true; eng->options.on_participant_info(participant, is_local, eng->options.ctx); } - return true; + break; // TODO: Only handle if needed case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: livekit_pb_session_description_t *answer = &res->message.answer; peer_handle_sdp(eng->pub_peer_handle, answer->sdp); - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: livekit_pb_session_description_t *offer = &res->message.offer; peer_handle_sdp(eng->sub_peer_handle, offer->sdp); - return true; + break; case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: livekit_pb_trickle_request_t *trickle = &res->message.trickle; char* candidate = NULL; if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { - return true; + break; } peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? eng->pub_peer_handle : eng->sub_peer_handle; peer_handle_ice_candidate(target_peer, candidate); free(candidate); - return true; + break; default: - return true; + break; } - return true; + break; case EV_SIG_STATE: if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; case EV_PEER_PUB_STATE: if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; case EV_PEER_SUB_STATE: if (ev->detail.state == CONNECTION_STATE_FAILED || ev->detail.state == CONNECTION_STATE_DISCONNECTED) { eng->state = ENGINE_STATE_BACKOFF; } - return true; + break; default: - return false; + break; } } -static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) +static void handle_state_backoff(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -741,7 +742,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); - return true; + break; } // TODO: Exponential backoff uint32_t backoff_ms = 1000; @@ -751,29 +752,29 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); xTimerStart(eng->timer, 0); - return true; + break; case EV_MAX_RETRIES_REACHED: eng->state = ENGINE_STATE_DISCONNECTED; - return true; + break; case EV_TIMER_EXP: eng->retry_count++; eng->state = ENGINE_STATE_CONNECTING; - return true; + break; case _EV_STATE_EXIT: xTimerStop(eng->timer, portMAX_DELAY); - return true; + break; default: - return false; + break; } } -static inline bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) +static inline void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) { switch (state) { - case ENGINE_STATE_DISCONNECTED: return handle_state_disconnected(eng, ev); - case ENGINE_STATE_CONNECTING: return handle_state_connecting(eng, ev); - case ENGINE_STATE_CONNECTED: return handle_state_connected(eng, ev); - case ENGINE_STATE_BACKOFF: return handle_state_backoff(eng, ev); + case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng, ev); break; + case ENGINE_STATE_CONNECTING: handle_state_connecting(eng, ev); break; + case ENGINE_STATE_CONNECTED: handle_state_connected(eng, ev); break; + case ENGINE_STATE_BACKOFF: handle_state_backoff(eng, ev); break; default: esp_system_abort("Unknown engine state"); } } From 0f719119698bad56190ae78a2d3390d04c210d74 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:59:20 +1000 Subject: [PATCH 22/81] Helper methods for data packets --- components/livekit/core/peer.c | 10 ++++------ components/livekit/core/protocol.c | 17 ++++++++++++++++- components/livekit/core/protocol.h | 18 +++++++++++++----- components/livekit/core/signaling.c | 2 +- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index e4aa173..e967f7a 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -224,15 +224,13 @@ static int on_data(esp_peer_data_frame_t *frame, void *ctx) return -1; } - livekit_pb_data_packet_t packet = {}; - pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)frame->data, frame->size); - if (!pb_decode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, &packet)) { - ESP_LOGE(TAG(peer), "Failed to decode data packet: %s", stream.errmsg); + livekit_pb_data_packet_t packet; + if (!protocol_data_packet_decode((const uint8_t *)frame->data, frame->size, &packet)) { + ESP_LOGE(TAG(peer), "Failed to decode data packet"); return -1; } - peer->options.on_packet_received(&packet, peer->options.ctx); - pb_release(LIVEKIT_PB_DATA_PACKET_FIELDS, &packet); + protocol_data_packet_free(&packet); return 0; } diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index b058a17..b3007d4 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -4,7 +4,22 @@ static const char *TAG = "livekit_protocol"; -bool protocol_signal_res_decode(const char *buf, size_t len, livekit_pb_signal_response_t* out) +bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out) +{ + pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); + if (!pb_decode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, out)) { + ESP_LOGE(TAG, "Failed to decode data packet: %s", stream.errmsg); + return false; + } + return true; +} + +void protocol_data_packet_free(livekit_pb_data_packet_t *packet) +{ + pb_release(LIVEKIT_PB_DATA_PACKET_FIELDS, packet); +} + +bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, out)) { diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 9402566..8071fc7 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -16,14 +16,22 @@ extern "C" { /// Server identifier (SID) type. typedef char livekit_pb_sid_t[16]; -/// Decodes a signal response. -bool protocol_signal_res_decode(const char *buf, size_t len, livekit_pb_signal_response_t* out); +/// Decodes a data packet. +/// +/// When the packet is no longer needed, free using `protocol_data_packet_free`. +/// +bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out); -/// Frees a signal response. +/// Frees a data packet. +void protocol_data_packet_free(livekit_pb_data_packet_t *packet); + +/// Decodes a signal response. /// -/// Always use this to discard signal responses, even for messages types that do not have -/// dynamic fields as they may be added in the future. +/// When the response is no longer needed, free using `protocol_signal_res_free`. /// +bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out); + +/// Frees a signal response. void protocol_signal_res_free(livekit_pb_signal_response_t *res); /// Extract ICE candidate string from a trickle request. diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 0d11c85..c4638aa 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -146,7 +146,7 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void } if (data->data_len < 1) break; livekit_pb_signal_response_t res; - if (!protocol_signal_res_decode(data->data_ptr, data->data_len, &res)) { + if (!protocol_signal_res_decode((const uint8_t *)data->data_ptr, data->data_len, &res)) { break; } if (!res_middleware(sg, &res)) { From 2a7a1eb1a276088c3af3cd777a26cc8dd2c90a59 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:09:50 +1000 Subject: [PATCH 23/81] Allow engine to take ownership of data packet --- components/livekit/core/engine.c | 6 ++++-- components/livekit/core/peer.c | 8 +++++--- components/livekit/core/peer.h | 11 ++++++++--- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index f4366d5..0578354 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -328,12 +328,14 @@ static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) // MARK: - Common peer event handlers -static void on_peer_packet_received(livekit_pb_data_packet_t* packet, void *ctx) +static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) { engine_t *eng = (engine_t *)ctx; + // TODO: Process through state machine. if (eng->options.on_data_packet) { eng->options.on_data_packet(packet, eng->options.ctx); } + return false; } static void on_peer_ice_candidate(const char *candidate, void *ctx) @@ -446,8 +448,8 @@ static bool establish_peer_connections(engine_t *eng) peer_options_t options = { .force_relay = eng->force_relay, .media = &eng->options.media, + .on_data_packet = on_peer_data_packet, .on_ice_candidate = on_peer_ice_candidate, - .on_packet_received = on_peer_packet_received, .ctx = eng }; diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index e967f7a..64524fb 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -215,7 +215,7 @@ static int on_data(esp_peer_data_frame_t *frame, void *ctx) peer_t *peer = (peer_t *)ctx; ESP_LOGD(TAG(peer), "Data received: size=%d, stream_id=%d", frame->size, frame->stream_id); - if (peer->options.on_packet_received == NULL) { + if (peer->options.on_data_packet == NULL) { ESP_LOGE(TAG(peer), "Packet received handler is not set"); return -1; } @@ -229,8 +229,10 @@ static int on_data(esp_peer_data_frame_t *frame, void *ctx) ESP_LOGE(TAG(peer), "Failed to decode data packet"); return -1; } - peer->options.on_packet_received(&packet, peer->options.ctx); - protocol_data_packet_free(&packet); + if (!peer->options.on_data_packet(&packet, peer->options.ctx)) { + // Ownership was not taken. + protocol_data_packet_free(&packet); + } return 0; } diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index cfd97f4..9f6d41f 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -41,6 +41,14 @@ typedef struct { /// Invoked when the peer's connection state changes. void (*on_state_changed)(connection_state_t state, void *ctx); + /// Invoked when a data packet is received over the data channel. + /// + /// The receiver returns true to take ownership of the packet. If + /// ownership is not taken (false), the packet will be freed with + /// `protocol_data_packet_free` internally. + /// + bool (*on_data_packet)(livekit_pb_data_packet_t* packet, void *ctx); + /// Invoked when an SDP message is available. This can be either /// an offer or answer depending on target configuration. void (*on_sdp)(const char *sdp, void *ctx); @@ -48,9 +56,6 @@ typedef struct { /// Invoked when a new ICE candidate is available. void (*on_ice_candidate)(const char *candidate, void *ctx); - /// Invoked when a data packet is received over the data channel. - void (*on_packet_received)(livekit_pb_data_packet_t* packet, void *ctx); - /// Invoked when information about an incoming audio stream is available. void (*on_audio_info)(esp_peer_audio_stream_info_t* info, void *ctx); From fcca62b47a30d81bb0dd7ee39505d4de3512f71c Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:47:26 +1000 Subject: [PATCH 24/81] Process data packets via state machine --- components/livekit/core/engine.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 0578354..afda852 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -25,6 +25,7 @@ typedef enum { EV_SIG_RES, /// Signal response received. EV_PEER_PUB_STATE, /// Publisher peer state changed. EV_PEER_SUB_STATE, /// Subscriber peer state changed. + EV_PEER_DATA_PACKET, /// Peer received data packet. EV_TIMER_EXP, /// Timer expired. EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. _EV_STATE_ENTER, @@ -45,6 +46,9 @@ typedef struct { /// Detail for `EV_SIG_RES`. livekit_pb_signal_response_t res; + /// Detail for `EV_PEER_DATA_PACKET`. + livekit_pb_data_packet_t data_packet; + /// Detail for `EV_SIG_STATE`, `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. connection_state_t state; } detail; @@ -87,6 +91,9 @@ static void event_free(engine_event_t *ev) if (ev->detail.cmd_connect.token != NULL) free(ev->detail.cmd_connect.token); break; + case EV_PEER_DATA_PACKET: + protocol_data_packet_free(&ev->detail.data_packet); + break; case EV_SIG_RES: protocol_signal_res_free(&ev->detail.res); break; @@ -331,11 +338,11 @@ static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) { engine_t *eng = (engine_t *)ctx; - // TODO: Process through state machine. - if (eng->options.on_data_packet) { - eng->options.on_data_packet(packet, eng->options.ctx); - } - return false; + engine_event_t ev = { + .type = EV_PEER_DATA_PACKET, + .detail.data_packet = *packet + }; + return xQueueSend(eng->event_queue, &ev, 0) == pdPASS; } static void on_peer_ice_candidate(const char *candidate, void *ctx) @@ -655,6 +662,12 @@ static void handle_state_connected(engine_t *eng, const engine_event_t *ev) case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); break; + case EV_PEER_DATA_PACKET: + livekit_pb_data_packet_t *packet = &ev->detail.data_packet; + if (eng->options.on_data_packet) { + eng->options.on_data_packet(packet, eng->options.ctx); + } + break; case EV_SIG_RES: livekit_pb_signal_response_t *res = &ev->detail.res; switch (ev->detail.res.which_message) { From 8b2b1db9b7eb6eef5e0a1a7934a678a127c8875e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:58:01 +1000 Subject: [PATCH 25/81] Notify of engine state change --- components/livekit/core/engine.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index afda852..449e9f3 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -516,10 +516,24 @@ static void engine_task(void *arg) state = eng->state; handle_state(eng, &(engine_event_t){ .type = _EV_STATE_EXIT }, state); assert(eng->state == state); - handle_state(eng, &(engine_event_t){ .type = _EV_STATE_ENTER }, eng->state); assert(eng->state == state); - // TODO: Notify of state change + + // Map engine state to external state, notify of state change. + if (eng->options.on_state_changed) { + connection_state_t ext_state; + switch (eng->state) { + case ENGINE_STATE_DISCONNECTED: ext_state = CONNECTION_STATE_DISCONNECTED; break; + case ENGINE_STATE_CONNECTING: ext_state = eng->retry_count > 0 ? + CONNECTION_STATE_RECONNECTING : + CONNECTION_STATE_CONNECTING; + break; + case ENGINE_STATE_BACKOFF: ext_state = CONNECTION_STATE_RECONNECTING; break; + case ENGINE_STATE_CONNECTED: ext_state = CONNECTION_STATE_CONNECTED; break; + default: ext_state = CONNECTION_STATE_DISCONNECTED; break; + } + eng->options.on_state_changed(ext_state, eng->options.ctx); + } } } From d47510a1932435eb0ab6ec34ce51802c1188c3a2 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:03:22 +1000 Subject: [PATCH 26/81] Add documentation --- components/livekit/core/engine.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 449e9f3..36fc7a5 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -342,6 +342,8 @@ static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) .type = EV_PEER_DATA_PACKET, .detail.data_packet = *packet }; + // Returning true indicates ownership of the packet; it will be freed when + // the queue is processed or flushed. return xQueueSend(eng->event_queue, &ev, 0) == pdPASS; } From 8569b417eeea0372bdd4dae54114fe9e87444e8e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:30:46 +1000 Subject: [PATCH 27/81] Allow handlers to take ownership of events --- components/livekit/core/engine.c | 82 +++++++++++++++++--------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 36fc7a5..99db0e8 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -80,27 +80,6 @@ typedef struct { int retry_count; } engine_t; -/// Frees engine events that contain dynamic fields. -static void event_free(engine_event_t *ev) -{ - if (ev == NULL) return; - switch (ev->type) { - case EV_CMD_CONNECT: - if (ev->detail.cmd_connect.server_url != NULL) - free(ev->detail.cmd_connect.server_url); - if (ev->detail.cmd_connect.token != NULL) - free(ev->detail.cmd_connect.token); - break; - case EV_PEER_DATA_PACKET: - protocol_data_packet_free(&ev->detail.data_packet); - break; - case EV_SIG_RES: - protocol_signal_res_free(&ev->detail.res); - break; - default: break; - } -} - // MARK: - Subscribed media /// Converts `esp_peer_audio_codec_t` to equivalent `av_render_audio_codec_t` value. @@ -488,8 +467,9 @@ static bool establish_peer_connections(engine_t *eng) // MARK: - Connection state machine -static void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); +static bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); static void flush_event_queue(engine_t *eng); +static void event_free(engine_event_t *ev); static void engine_task(void *arg) { @@ -506,9 +486,12 @@ static void engine_task(void *arg) engine_state_t state = eng->state; // Invoke the handler for the current state, passing the event that woke up the - // state machine, then free the event once the handler has completed. - handle_state(eng, &ev, state); - event_free(&ev); + // state machine. If the handler returns true, it takes ownership of the event + // and is responsible for freeing it, otherwise, it will be freed after the handler + // returns. + if (!handle_state(eng, &ev, state)) { + event_free(&ev); + } // If the state changed, invoke the exit handler for the old state, // the enter handler for the new state, and notify. @@ -544,7 +527,7 @@ static void engine_task(void *arg) vTaskDelete(NULL); } -static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) +static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -561,17 +544,17 @@ static void handle_state_disconnected(engine_t *eng, const engine_event_t *ev) case EV_CMD_CONNECT: if (eng->server_url != NULL) free(eng->server_url); if (eng->token != NULL) free(eng->token); - eng->server_url = strdup(ev->detail.cmd_connect.server_url); - eng->token = strdup(ev->detail.cmd_connect.token); - + eng->server_url = ev->detail.cmd_connect.server_url; + eng->token = ev->detail.cmd_connect.token; eng->state = ENGINE_STATE_CONNECTING; - break; + return true; default: break; } + return false; } -static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) +static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -662,9 +645,10 @@ static void handle_state_connecting(engine_t *eng, const engine_event_t *ev) default: break; } + return false; } -static void handle_state_connected(engine_t *eng, const engine_event_t *ev) +static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -760,9 +744,10 @@ static void handle_state_connected(engine_t *eng, const engine_event_t *ev) default: break; } + return false; } -static void handle_state_backoff(engine_t *eng, const engine_event_t *ev) +static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: @@ -797,15 +782,16 @@ static void handle_state_backoff(engine_t *eng, const engine_event_t *ev) default: break; } + return false; } -static inline void handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) +static inline bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) { switch (state) { - case ENGINE_STATE_DISCONNECTED: handle_state_disconnected(eng, ev); break; - case ENGINE_STATE_CONNECTING: handle_state_connecting(eng, ev); break; - case ENGINE_STATE_CONNECTED: handle_state_connected(eng, ev); break; - case ENGINE_STATE_BACKOFF: handle_state_backoff(eng, ev); break; + case ENGINE_STATE_DISCONNECTED: return handle_state_disconnected(eng, ev); + case ENGINE_STATE_CONNECTING: return handle_state_connecting(eng, ev); + case ENGINE_STATE_CONNECTED: return handle_state_connected(eng, ev); + case ENGINE_STATE_BACKOFF: return handle_state_backoff(eng, ev); default: esp_system_abort("Unknown engine state"); } } @@ -821,6 +807,26 @@ static void flush_event_queue(engine_t *eng) ESP_LOGI(TAG, "Flushed %d events", count); } +static void event_free(engine_event_t *ev) +{ + if (ev == NULL) return; + switch (ev->type) { + case EV_CMD_CONNECT: + if (ev->detail.cmd_connect.server_url != NULL) + free(ev->detail.cmd_connect.server_url); + if (ev->detail.cmd_connect.token != NULL) + free(ev->detail.cmd_connect.token); + break; + case EV_PEER_DATA_PACKET: + protocol_data_packet_free(&ev->detail.data_packet); + break; + case EV_SIG_RES: + protocol_signal_res_free(&ev->detail.res); + break; + default: break; + } +} + // MARK: - Public API engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) From 5d64e933e8c79608b32f0bb6903792bff109beaf Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:07:54 +1000 Subject: [PATCH 28/81] Exponential backoff --- components/livekit/core/engine.c | 14 +++++++------- components/livekit/core/utils.c | 16 +++++++++++++++- components/livekit/core/utils.h | 7 +++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 99db0e8..0a06f4d 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -10,6 +10,7 @@ #include "url.h" #include "signaling.h" #include "peer.h" +#include "utils.h" #include "engine.h" // MARK: - Constants @@ -77,7 +78,7 @@ typedef struct { QueueHandle_t event_queue; TimerHandle_t timer; bool is_running; - int retry_count; + uint16_t retry_count; } engine_t; // MARK: - Subscribed media @@ -755,16 +756,16 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) signal_close(eng->signal_handle); destroy_peer_connections(eng); + eng->retry_count++; + if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); break; } - // TODO: Exponential backoff - uint32_t backoff_ms = 1000; - - ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu32 "ms", - eng->retry_count + 1, CONFIG_LK_MAX_RETRIES, backoff_ms); + uint16_t backoff_ms = backoff_ms_for_attempt(eng->retry_count); + ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu16 "ms", + eng->retry_count, CONFIG_LK_MAX_RETRIES, backoff_ms); xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); xTimerStart(eng->timer, 0); @@ -773,7 +774,6 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_TIMER_EXP: - eng->retry_count++; eng->state = ENGINE_STATE_CONNECTING; break; case _EV_STATE_EXIT: diff --git a/components/livekit/core/utils.c b/components/livekit/core/utils.c index 4f6339e..f47ddfd 100644 --- a/components/livekit/core/utils.c +++ b/components/livekit/core/utils.c @@ -1,9 +1,23 @@ #include +#include +#include "esp_random.h" #include "utils.h" -int64_t get_unix_time_ms(void) { +#define MAX_BACKOFF_MS 7000 + +int64_t get_unix_time_ms(void) +{ struct timeval tv; gettimeofday(&tv, NULL); return (int64_t)tv.tv_sec * 1000LL + (tv.tv_usec / 1000LL); +} + +uint16_t backoff_ms_for_attempt(uint16_t attempt) +{ + if (attempt == 0) return 0; + uint16_t base = 100 << attempt; // 100 * (2^attempt) + uint16_t rand = (uint16_t)(esp_random() % 1001); // range [0, 1000] + uint16_t total = base + rand; + return total > MAX_BACKOFF_MS ? MAX_BACKOFF_MS : total; } \ No newline at end of file diff --git a/components/livekit/core/utils.h b/components/livekit/core/utils.h index 4b0158c..ec96d9d 100644 --- a/components/livekit/core/utils.h +++ b/components/livekit/core/utils.h @@ -9,6 +9,13 @@ extern "C" { int64_t get_unix_time_ms(void); +/// Returns the backoff time in milliseconds for the given attempt number. +/// +/// Uses an exponential function with a random jitter to calculate the backoff time +/// with the value limited to an upper bound. +/// +uint16_t backoff_ms_for_attempt(uint16_t attempt); + #ifdef __cplusplus } #endif From da8e6af3bdc7c490d7fc4b80a46516d58ab751ea Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:58:33 +1000 Subject: [PATCH 29/81] Create helper to enqueue events --- components/livekit/core/engine.c | 33 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 0a06f4d..04a0992 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -81,6 +81,17 @@ typedef struct { uint16_t retry_count; } engine_t; +static inline bool event_enqueue(engine_t *eng, engine_event_t *ev, bool send_to_front) +{ + bool enqueued = (send_to_front ? + xQueueSendToFront(eng->event_queue, ev, 0) : + xQueueSend(eng->event_queue, ev, 0)) == pdPASS; + if (!enqueued) { + ESP_LOGE(TAG, "Failed to enqueue event: type=%d", ev->type); + } + return enqueued; +} + // MARK: - Subscribed media /// Converts `esp_peer_audio_codec_t` to equivalent `av_render_audio_codec_t` value. @@ -295,7 +306,7 @@ static void on_signal_state_changed(connection_state_t state, void *ctx) .type = EV_SIG_STATE, .detail.state = state }; - xQueueSendToFront(eng->event_queue, &ev, 0); + event_enqueue(eng, &ev, true); } static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) @@ -307,10 +318,8 @@ static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) }; // Returning true takes ownership of the response; it will be freed later when the // queue is processed or flushed. - if (res->which_message == LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG) { - return xQueueSendToFront(eng->event_queue, &ev, 0) == pdPASS; - } - return xQueueSend(eng->event_queue, &ev, 0) == pdPASS; + bool send_to_front = res->which_message == LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG; + return event_enqueue(eng, &ev, send_to_front); } // MARK: - Common peer event handlers @@ -324,7 +333,7 @@ static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) }; // Returning true indicates ownership of the packet; it will be freed when // the queue is processed or flushed. - return xQueueSend(eng->event_queue, &ev, 0) == pdPASS; + return event_enqueue(eng, &ev, false); } static void on_peer_ice_candidate(const char *candidate, void *ctx) @@ -341,7 +350,7 @@ static void on_peer_pub_state_changed(connection_state_t state, void *ctx) .type = EV_PEER_PUB_STATE, .detail.state = state }; - xQueueSendToFront(eng->event_queue, &ev, 0); + event_enqueue(eng, &ev, true); } static void on_peer_pub_offer(const char *sdp, void *ctx) @@ -359,7 +368,7 @@ static void on_peer_sub_state_changed(connection_state_t state, void *ctx) .type = EV_PEER_SUB_STATE, .detail.state = state }; - xQueueSendToFront(eng->event_queue, &ev, 0); + event_enqueue(eng, &ev, true); } static void on_peer_sub_answer(const char *sdp, void *ctx) @@ -403,7 +412,7 @@ static void on_timer_expired(TimerHandle_t timer) { engine_t *eng = (engine_t *)pvTimerGetTimerID(timer); engine_event_t ev = { .type = EV_TIMER_EXP }; - xQueueSend(eng->event_queue, &ev, 0); + event_enqueue(eng, &ev, true); } // MARK: - Peer lifecycle @@ -760,7 +769,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { ESP_LOGW(TAG, "Max retries reached"); - xQueueSendToFront(eng->event_queue, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, 0); + event_enqueue(eng, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, true); break; } uint16_t backoff_ms = backoff_ms_for_attempt(eng->retry_count); @@ -944,7 +953,7 @@ engine_err_t engine_connect(engine_handle_t handle, const char* server_url, cons .type = EV_CMD_CONNECT, .detail.cmd_connect = { .server_url = strdup(server_url), .token = strdup(token) } }; - if (xQueueSendToFront(eng->event_queue, &ev, 0) != pdPASS) { + if (!event_enqueue(eng, &ev, true)) { return ENGINE_ERR_OTHER; } return ENGINE_ERR_NONE; @@ -958,7 +967,7 @@ engine_err_t engine_close(engine_handle_t handle) engine_t *eng = (engine_t *)handle; engine_event_t ev = { .type = EV_CMD_CLOSE }; - if (xQueueSendToFront(eng->event_queue, &ev, 0) != pdPASS) { + if (!event_enqueue(eng, &ev, true)) { return ENGINE_ERR_OTHER; } return ENGINE_ERR_NONE; From eb9a9792a2743debcf9c0f2f6bc5db9b8d73edbb Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:02:41 +1000 Subject: [PATCH 30/81] Zero initialize protobuf structs --- components/livekit/core/peer.c | 2 +- components/livekit/core/signaling.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index 64524fb..edcc675 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -224,7 +224,7 @@ static int on_data(esp_peer_data_frame_t *frame, void *ctx) return -1; } - livekit_pb_data_packet_t packet; + livekit_pb_data_packet_t packet = {}; if (!protocol_data_packet_decode((const uint8_t *)frame->data, frame->size, &packet)) { ESP_LOGE(TAG(peer), "Failed to decode data packet"); return -1; diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index c4638aa..3a154f3 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -145,7 +145,7 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void break; } if (data->data_len < 1) break; - livekit_pb_signal_response_t res; + livekit_pb_signal_response_t res = {}; if (!protocol_signal_res_decode((const uint8_t *)data->data_ptr, data->data_len, &res)) { break; } From 22f5722342143408f81d0c731854d98db234d4e6 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:07:11 +1000 Subject: [PATCH 31/81] Ignore non-binary messages silently --- components/livekit/core/signaling.c | 1 - 1 file changed, 1 deletion(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 3a154f3..9b884b4 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -141,7 +141,6 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void break; case WEBSOCKET_EVENT_DATA: if (data->op_code != WS_TRANSPORT_OPCODES_BINARY) { - ESP_LOGW(TAG, "Received non-binary message"); break; } if (data->data_len < 1) break; From 4f13b19c6081ebc130eea19834cdb951bf9b3688 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:25:41 +1000 Subject: [PATCH 32/81] Make engine state internal --- components/livekit/core/engine.c | 8 ++++++++ components/livekit/core/engine.h | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 04a0992..251e9a4 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -18,6 +18,14 @@ static const char* TAG = "livekit_engine"; // MARK: - Type definitions +/// Engine state machine state. +typedef enum { + ENGINE_STATE_DISCONNECTED, + ENGINE_STATE_CONNECTING, + ENGINE_STATE_CONNECTED, + ENGINE_STATE_BACKOFF +} engine_state_t; + /// Type of event processed by the engine state machine. typedef enum { EV_CMD_CONNECT, /// User-initiated connect. diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 99e6e31..5387a64 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -18,13 +18,6 @@ extern "C" { /// Handle to an engine instance. typedef void *engine_handle_t; -typedef enum { - ENGINE_STATE_DISCONNECTED, - ENGINE_STATE_CONNECTING, - ENGINE_STATE_CONNECTED, - ENGINE_STATE_BACKOFF -} engine_state_t; - typedef enum { ENGINE_ERR_NONE = 0, ENGINE_ERR_INVALID_ARG = -1, From df1ff7f3ec45eac355594b00db31c3b85474eaca Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:26:16 +1000 Subject: [PATCH 33/81] Document --- components/livekit/core/engine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 251e9a4..1868602 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -37,8 +37,8 @@ typedef enum { EV_PEER_DATA_PACKET, /// Peer received data packet. EV_TIMER_EXP, /// Timer expired. EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. - _EV_STATE_ENTER, - _EV_STATE_EXIT, + _EV_STATE_ENTER, /// State enter hook (internal). + _EV_STATE_EXIT, /// State exit hook (internal). } engine_event_type_t; /// An event processed by the engine state machine. From 0bd3c70670755666861e8641e90a1c0dcef8ad6a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 21 Aug 2025 15:01:26 +1000 Subject: [PATCH 34/81] Handle and expose failure reason - Do not retry for client errors - Expose failure reason through public API --- components/livekit/core/engine.c | 270 ++++++++++++--------- components/livekit/core/engine.h | 6 +- components/livekit/core/livekit.c | 58 ++--- components/livekit/core/signaling.c | 49 ++-- components/livekit/core/signaling.h | 33 +++ components/livekit/include/livekit.h | 28 ++- components/livekit/include/livekit_types.h | 67 +++++ examples/minimal/main/example.c | 6 +- 8 files changed, 345 insertions(+), 172 deletions(-) create mode 100644 components/livekit/include/livekit_types.h diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 1868602..b3dc98e 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -87,18 +87,10 @@ typedef struct { TimerHandle_t timer; bool is_running; uint16_t retry_count; + livekit_failure_reason_t failure_reason; } engine_t; -static inline bool event_enqueue(engine_t *eng, engine_event_t *ev, bool send_to_front) -{ - bool enqueued = (send_to_front ? - xQueueSendToFront(eng->event_queue, ev, 0) : - xQueueSend(eng->event_queue, ev, 0)) == pdPASS; - if (!enqueued) { - ESP_LOGE(TAG, "Failed to enqueue event: type=%d", ev->type); - } - return enqueued; -} +static bool event_enqueue(engine_t *eng, engine_event_t *ev, bool send_to_front); // MARK: - Subscribed media @@ -483,68 +475,89 @@ static bool establish_peer_connections(engine_t *eng) return true; } -// MARK: - Connection state machine - -static bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state); -static void flush_event_queue(engine_t *eng); -static void event_free(engine_event_t *ev); +// MARK: - FSM helpers -static void engine_task(void *arg) +/// Return the external state that should be reported based on the engine's current state. +/// +/// This is necessary because the engine FSM's states do not map 1:1 with the states +/// exposed in the public room API. +/// +static inline livekit_connection_state_t map_engine_state(engine_t *eng) { - engine_t *eng = (engine_t *)arg; - while (eng->is_running) { - engine_event_t ev; - if (!xQueueReceive(eng->event_queue, &ev, portMAX_DELAY)) { - ESP_LOGE(TAG, "Failed to receive event"); - continue; - } - assert(ev.type != _EV_STATE_ENTER && ev.type != _EV_STATE_EXIT); - ESP_LOGI(TAG, "Event: %d", ev.type); - - engine_state_t state = eng->state; - - // Invoke the handler for the current state, passing the event that woke up the - // state machine. If the handler returns true, it takes ownership of the event - // and is responsible for freeing it, otherwise, it will be freed after the handler - // returns. - if (!handle_state(eng, &ev, state)) { - event_free(&ev); - } + switch (eng->state) { + case ENGINE_STATE_DISCONNECTED: + // Engine state machine doesn't have a discrete failed state + return eng->failure_reason == LIVEKIT_FAILURE_REASON_NONE ? + LIVEKIT_CONNECTION_STATE_DISCONNECTED : + LIVEKIT_CONNECTION_STATE_FAILED; + case ENGINE_STATE_CONNECTING: + // Should only report connecting for initial connection attempt. + return eng->retry_count <= 0 ? LIVEKIT_CONNECTION_STATE_CONNECTING : -1; + case ENGINE_STATE_BACKOFF: + return LIVEKIT_CONNECTION_STATE_RECONNECTING; + case ENGINE_STATE_CONNECTED: + return LIVEKIT_CONNECTION_STATE_CONNECTED; + default: + return LIVEKIT_CONNECTION_STATE_DISCONNECTED; + } +} - // If the state changed, invoke the exit handler for the old state, - // the enter handler for the new state, and notify. - if (eng->state != state) { - ESP_LOGI(TAG, "State changed: %d -> %d", state, eng->state); +/// Map a signal failure reason to the failure reason exposed in the public room API. +static livekit_failure_reason_t map_signal_failure_reason(signal_failure_reason_t reason) +{ + switch (reason) { + case SIGNAL_FAILURE_REASON_UNREACHABLE: return LIVEKIT_FAILURE_REASON_UNREACHABLE; + case SIGNAL_FAILURE_REASON_BAD_TOKEN: return LIVEKIT_FAILURE_REASON_BAD_TOKEN; + case SIGNAL_FAILURE_REASON_UNAUTHORIZED: return LIVEKIT_FAILURE_REASON_UNAUTHORIZED; + default: return LIVEKIT_FAILURE_REASON_OTHER; + } +} - state = eng->state; - handle_state(eng, &(engine_event_t){ .type = _EV_STATE_EXIT }, state); - assert(eng->state == state); - handle_state(eng, &(engine_event_t){ .type = _EV_STATE_ENTER }, eng->state); - assert(eng->state == state); +/// Frees an event's dynamically allocated fields (if any). +static void event_free(engine_event_t *ev) +{ + if (ev == NULL) return; + switch (ev->type) { + case EV_CMD_CONNECT: + if (ev->detail.cmd_connect.server_url != NULL) + free(ev->detail.cmd_connect.server_url); + if (ev->detail.cmd_connect.token != NULL) + free(ev->detail.cmd_connect.token); + break; + case EV_PEER_DATA_PACKET: + protocol_data_packet_free(&ev->detail.data_packet); + break; + case EV_SIG_RES: + protocol_signal_res_free(&ev->detail.res); + break; + default: break; + } +} - // Map engine state to external state, notify of state change. - if (eng->options.on_state_changed) { - connection_state_t ext_state; - switch (eng->state) { - case ENGINE_STATE_DISCONNECTED: ext_state = CONNECTION_STATE_DISCONNECTED; break; - case ENGINE_STATE_CONNECTING: ext_state = eng->retry_count > 0 ? - CONNECTION_STATE_RECONNECTING : - CONNECTION_STATE_CONNECTING; - break; - case ENGINE_STATE_BACKOFF: ext_state = CONNECTION_STATE_RECONNECTING; break; - case ENGINE_STATE_CONNECTED: ext_state = CONNECTION_STATE_CONNECTED; break; - default: ext_state = CONNECTION_STATE_DISCONNECTED; break; - } - eng->options.on_state_changed(ext_state, eng->options.ctx); - } - } +/// Enqueues an event. +static bool event_enqueue(engine_t *eng, engine_event_t *ev, bool send_to_front) +{ + bool enqueued = (send_to_front ? + xQueueSendToFront(eng->event_queue, ev, 0) : + xQueueSend(eng->event_queue, ev, 0)) == pdPASS; + if (!enqueued) { + ESP_LOGE(TAG, "Failed to enqueue event: type=%d", ev->type); } + return enqueued; +} - // Discard any remaining events in the queue before exiting. - flush_event_queue(eng); - vTaskDelete(NULL); +/// Dequeues all events from the queue and frees them. +static void flush_event_queue(engine_t *eng) +{ + engine_event_t ev; + while (xQueueReceive(eng->event_queue, &ev, 0) == pdPASS) { + event_free(&ev); + } } +// MARK: - FSM state handlers + +/// Handler for `ENGINE_STATE_DISCONNECTED`. static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { @@ -564,6 +577,7 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) if (eng->token != NULL) free(eng->token); eng->server_url = ev->detail.cmd_connect.server_url; eng->token = ev->detail.cmd_connect.token; + eng->failure_reason = LIVEKIT_FAILURE_REASON_NONE; eng->state = ENGINE_STATE_CONNECTING; return true; default: @@ -572,6 +586,7 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) return false; } +/// Handler for `ENGINE_STATE_CONNECTING`. static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { @@ -636,27 +651,28 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - // TODO: Check error code (4xx should go to disconnected) - if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { - eng->state = ENGINE_STATE_BACKOFF; + if (ev->detail.state == CONNECTION_STATE_FAILED) { + signal_failure_reason_t reason = signal_get_failure_reason(eng->signal_handle); + eng->failure_reason = map_signal_failure_reason(reason); + // Client errors should not trigger a reconnection + eng->state = (reason & SIGNAL_FAILURE_REASON_CLIENT_ANY) ? + ENGINE_STATE_DISCONNECTED : + ENGINE_STATE_BACKOFF; } break; case EV_PEER_PUB_STATE: - if (!eng->is_subscriber_primary && - ev->detail.state == CONNECTION_STATE_CONNECTED) { + if (!eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; - } else if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { + } else if (ev->detail.state == CONNECTION_STATE_FAILED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } break; case EV_PEER_SUB_STATE: - if (eng->is_subscriber_primary && - ev->detail.state == CONNECTION_STATE_CONNECTED) { + if (eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { eng->state = ENGINE_STATE_CONNECTED; - } else if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { + } else if (ev->detail.state == CONNECTION_STATE_FAILED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } break; @@ -666,11 +682,13 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) return false; } +/// Handler for `ENGINE_STATE_CONNECTED`. static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: eng->retry_count = 0; + eng->failure_reason = LIVEKIT_FAILURE_REASON_NONE; publish_tracks(eng); break; case EV_CMD_CLOSE: @@ -742,20 +760,20 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_UNREACHABLE; eng->state = ENGINE_STATE_BACKOFF; } break; case EV_PEER_PUB_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } break; case EV_PEER_SUB_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED || - ev->detail.state == CONNECTION_STATE_DISCONNECTED) { + if (ev->detail.state == CONNECTION_STATE_FAILED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } break; @@ -765,6 +783,7 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) return false; } +/// Handler for `ENGINE_STATE_BACKOFF`. static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { @@ -774,20 +793,20 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) destroy_peer_connections(eng); eng->retry_count++; - - if (eng->retry_count >= CONFIG_LK_MAX_RETRIES) { - ESP_LOGW(TAG, "Max retries reached"); + if (eng->retry_count > CONFIG_LK_MAX_RETRIES) { + // State changes within enter/exit are not allowed; enqueue event instead. event_enqueue(eng, &(engine_event_t){ .type = EV_MAX_RETRIES_REACHED }, true); break; } uint16_t backoff_ms = backoff_ms_for_attempt(eng->retry_count); - ESP_LOGI(TAG, "Attempting reconnect %d/%d in %" PRIu16 "ms", - eng->retry_count, CONFIG_LK_MAX_RETRIES, backoff_ms); + ESP_LOGI(TAG, "Reconnect in %" PRIu16 "ms: attempt=%d/%d, reason=%d", + backoff_ms, eng->retry_count, CONFIG_LK_MAX_RETRIES, eng->failure_reason); xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); xTimerStart(eng->timer, 0); break; case EV_MAX_RETRIES_REACHED: + eng->failure_reason = LIVEKIT_FAILURE_REASON_MAX_RETRIES; eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_TIMER_EXP: @@ -802,6 +821,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) return false; } +/// Invokes the handler for the given state. static inline bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_t state) { switch (state) { @@ -813,35 +833,52 @@ static inline bool handle_state(engine_t *eng, engine_event_t *ev, engine_state_ } } -static void flush_event_queue(engine_t *eng) -{ - engine_event_t ev; - int count = 0; - while (xQueueReceive(eng->event_queue, &ev, 0) == pdPASS) { - count++; - event_free(&ev); - } - ESP_LOGI(TAG, "Flushed %d events", count); -} +// MARK: - FSM task -static void event_free(engine_event_t *ev) +static void engine_task(void *arg) { - if (ev == NULL) return; - switch (ev->type) { - case EV_CMD_CONNECT: - if (ev->detail.cmd_connect.server_url != NULL) - free(ev->detail.cmd_connect.server_url); - if (ev->detail.cmd_connect.token != NULL) - free(ev->detail.cmd_connect.token); - break; - case EV_PEER_DATA_PACKET: - protocol_data_packet_free(&ev->detail.data_packet); - break; - case EV_SIG_RES: - protocol_signal_res_free(&ev->detail.res); - break; - default: break; + engine_t *eng = (engine_t *)arg; + while (eng->is_running) { + engine_event_t ev; + if (!xQueueReceive(eng->event_queue, &ev, portMAX_DELAY)) { + ESP_LOGE(TAG, "Failed to receive event"); + continue; + } + // Internal events are not allowed to be enqueued. + assert(ev.type != _EV_STATE_ENTER && ev.type != _EV_STATE_EXIT); + ESP_LOGD(TAG, "Event: type=%d", ev.type); + + engine_state_t state = eng->state; + + // Invoke the handler for the current state, passing the event that woke up the + // state machine. If the handler returns true, it takes ownership of the event + // and is responsible for freeing it, otherwise, it will be freed after the handler + // returns. + if (!handle_state(eng, &ev, state)) { + event_free(&ev); + } + + // If the state changed, invoke the exit handler for the old state, + // the enter handler for the new state, and notify. + if (eng->state != state) { + ESP_LOGD(TAG, "State changed: %d -> %d", state, eng->state); + + state = eng->state; + handle_state(eng, &(engine_event_t){ .type = _EV_STATE_EXIT }, state); + assert(eng->state == state); + handle_state(eng, &(engine_event_t){ .type = _EV_STATE_ENTER }, eng->state); + assert(eng->state == state); + + if (eng->options.on_state_changed) { + livekit_connection_state_t ext_state = map_engine_state(eng); + if (ext_state > 0) eng->options.on_state_changed(ext_state, eng->options.ctx); + } + } } + + // Discard any remaining events in the queue before exiting. + flush_event_queue(eng); + vTaskDelete(NULL); } // MARK: - Public API @@ -981,6 +1018,15 @@ engine_err_t engine_close(engine_handle_t handle) return ENGINE_ERR_NONE; } +livekit_failure_reason_t engine_get_failure_reason(engine_handle_t handle) +{ + if (handle == NULL) { + return LIVEKIT_FAILURE_REASON_NONE; + } + engine_t *eng = (engine_t *)handle; + return eng->failure_reason; +} + engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind) { if (handle == NULL) { diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 5387a64..4223145 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -6,6 +6,7 @@ #include "esp_capture.h" #include "av_render.h" +#include "livekit_types.h" #include "common.h" #include "protocol.h" @@ -39,7 +40,7 @@ typedef struct { typedef struct { void *ctx; - void (*on_state_changed)(connection_state_t state, void *ctx); + void (*on_state_changed)(livekit_connection_state_t state, void *ctx); void (*on_data_packet)(livekit_pb_data_packet_t* packet, void *ctx); void (*on_room_info)(const livekit_pb_room_t* info, void *ctx); void (*on_participant_info)(const livekit_pb_participant_info_t* info, bool is_local, void *ctx); @@ -60,6 +61,9 @@ engine_err_t engine_connect(engine_handle_t handle, const char* server_url, cons /// Close the engine. engine_err_t engine_close(engine_handle_t handle); +/// Returns the reason why the engine connection failed. +livekit_failure_reason_t engine_get_failure_reason(engine_handle_t handle); + /// Sends a data packet to the remote peer. engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind); diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index c761c35..01b1517 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -102,30 +102,12 @@ static void populate_media_options( media_options->renderer = sub_options->renderer; } -static void on_eng_state_changed(connection_state_t engine_state, void *ctx) +static void on_eng_state_changed(livekit_connection_state_t state, void *ctx) { livekit_room_t *room = (livekit_room_t *)ctx; - switch (engine_state) { - case CONNECTION_STATE_DISCONNECTED: - room->state = LIVEKIT_CONNECTION_STATE_DISCONNECTED; - break; - case CONNECTION_STATE_CONNECTED: - room->state = LIVEKIT_CONNECTION_STATE_CONNECTED; - break; - case CONNECTION_STATE_CONNECTING: - room->state = LIVEKIT_CONNECTION_STATE_CONNECTING; - break; - case CONNECTION_STATE_RECONNECTING: - room->state = LIVEKIT_CONNECTION_STATE_RECONNECTING; - break; - case CONNECTION_STATE_FAILED: - room->state = LIVEKIT_CONNECTION_STATE_FAILED; - break; - default: - return; - } + room->state = state; if (room->options.on_state_changed != NULL) { - room->options.on_state_changed(room->state, room->options.ctx); + room->options.on_state_changed(state, room->options.ctx); } } @@ -304,15 +286,37 @@ livekit_connection_state_t livekit_room_get_state(livekit_room_handle_t handle) const char* livekit_connection_state_str(livekit_connection_state_t state) { switch (state) { - case LIVEKIT_CONNECTION_STATE_DISCONNECTED: return "disconnected"; - case LIVEKIT_CONNECTION_STATE_CONNECTING: return "connecting"; - case LIVEKIT_CONNECTION_STATE_CONNECTED: return "connected"; - case LIVEKIT_CONNECTION_STATE_RECONNECTING: return "reconnecting"; - case LIVEKIT_CONNECTION_STATE_FAILED: return "failed"; - default: return "unknown"; + case LIVEKIT_CONNECTION_STATE_DISCONNECTED: return "Disconnected"; + case LIVEKIT_CONNECTION_STATE_CONNECTING: return "Connecting"; + case LIVEKIT_CONNECTION_STATE_CONNECTED: return "Connected"; + case LIVEKIT_CONNECTION_STATE_RECONNECTING: return "Reconnecting"; + case LIVEKIT_CONNECTION_STATE_FAILED: return "Failed"; + default: return "Unknown"; } } +const char* livekit_failure_reason_str(livekit_failure_reason_t reason) +{ + switch (reason) { + case LIVEKIT_FAILURE_REASON_NONE: return "None"; + case LIVEKIT_FAILURE_REASON_UNREACHABLE: return "Unreachable"; + case LIVEKIT_FAILURE_REASON_BAD_TOKEN: return "Bad Token"; + case LIVEKIT_FAILURE_REASON_UNAUTHORIZED: return "Unauthorized"; + case LIVEKIT_FAILURE_REASON_RTC: return "RTC"; + case LIVEKIT_FAILURE_REASON_MAX_RETRIES: return "Max Retries"; + default: return "Other"; + } +} + +livekit_failure_reason_t livekit_room_get_failure_reason(livekit_room_handle_t handle) +{ + if (handle == NULL) { + return LIVEKIT_FAILURE_REASON_NONE; + } + livekit_room_t *room = (livekit_room_t *)handle; + return engine_get_failure_reason(room->engine); +} + livekit_err_t livekit_room_publish_data(livekit_room_handle_t handle, livekit_data_publish_options_t *options) { if (handle == NULL || options == NULL || options->payload == NULL) { diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 9b884b4..37d5aa1 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -28,17 +28,22 @@ static const char *TAG = "livekit_signaling"; typedef struct { esp_websocket_client_handle_t ws; signal_options_t options; - esp_timer_handle_t ping_timer; + esp_timer_handle_t ping_timer; + signal_failure_reason_t failure_reason; - int32_t ping_interval_ms; - int32_t ping_timeout_ms; - int64_t rtt; + int32_t ping_interval_ms; + int32_t ping_timeout_ms; + int64_t rtt; } signal_t; -static void log_error_if_nonzero(const char *message, int error_code) +static inline signal_failure_reason_t failure_reason_from_http_status(int status) { - if (error_code != 0) { - ESP_LOGE(TAG, "Last error %s: 0x%x", message, error_code); + switch (status) { + case 400: return SIGNAL_FAILURE_REASON_BAD_TOKEN; + case 401: return SIGNAL_FAILURE_REASON_UNAUTHORIZED; + default: return status > 400 && status < 500 ? + SIGNAL_FAILURE_REASON_CLIENT_OTHER : + SIGNAL_FAILURE_REASON_INTERNAL; } } @@ -123,6 +128,9 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; switch (event_id) { + case WEBSOCKET_EVENT_BEFORE_CONNECT: + sg->failure_reason = SIGNAL_FAILURE_REASON_NONE; + break; case WEBSOCKET_EVENT_CONNECTED: ESP_LOGD(TAG, "Signaling connected"); sg->options.on_state_changed(CONNECTION_STATE_CONNECTED, sg->options.ctx); @@ -130,13 +138,6 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void case WEBSOCKET_EVENT_DISCONNECTED: ESP_LOGD(TAG, "Signaling disconnected"); esp_timer_stop(sg->ping_timer); - - log_error_if_nonzero("HTTP status code", data->error_handle.esp_ws_handshake_status_code); - if (data->error_handle.error_type == WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT) { - log_error_if_nonzero("reported from esp-tls", data->error_handle.esp_tls_last_esp_err); - log_error_if_nonzero("reported from tls stack", data->error_handle.esp_tls_stack_err); - log_error_if_nonzero("captured as transport's socket errno", data->error_handle.esp_transport_sock_errno); - } sg->options.on_state_changed(CONNECTION_STATE_DISCONNECTED, sg->options.ctx); break; case WEBSOCKET_EVENT_DATA: @@ -159,15 +160,12 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void } break; case WEBSOCKET_EVENT_ERROR: - ESP_LOGE(TAG, "Failed to connect to server"); esp_timer_stop(sg->ping_timer); - log_error_if_nonzero("HTTP status code", data->error_handle.esp_ws_handshake_status_code); - if (data->error_handle.error_type == WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT) { - log_error_if_nonzero("reported from esp-tls", data->error_handle.esp_tls_last_esp_err); - log_error_if_nonzero("reported from tls stack", data->error_handle.esp_tls_stack_err); - log_error_if_nonzero("captured as transport's socket errno", data->error_handle.esp_transport_sock_errno); - } + int http_status = data->error_handle.esp_ws_handshake_status_code; + sg->failure_reason = http_status != 0 ? + failure_reason_from_http_status(http_status) : + SIGNAL_FAILURE_REASON_UNREACHABLE; sg->options.on_state_changed(CONNECTION_STATE_FAILED, sg->options.ctx); break; default: break; @@ -281,6 +279,15 @@ signal_err_t signal_close(signal_handle_t handle) return SIGNAL_ERR_NONE; } +signal_failure_reason_t signal_get_failure_reason(signal_handle_t handle) +{ + if (handle == NULL) { + return SIGNAL_FAILURE_REASON_NONE; + } + signal_t *sg = (signal_t *)handle; + return sg->failure_reason; +} + signal_err_t signal_send_leave(signal_handle_t handle) { if (handle == NULL) { diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index af16ce7..710962d 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -21,6 +21,31 @@ typedef enum { // TODO: Add more error cases as needed } signal_err_t; +/// Reason why signal connection failed. +typedef enum { + /// No failure has occurred. + SIGNAL_FAILURE_REASON_NONE = 0, + + /// Server unreachable. + SIGNAL_FAILURE_REASON_UNREACHABLE = 1 << 0, + + /// Token is malformed. + SIGNAL_FAILURE_REASON_BAD_TOKEN = 1 << 1, + + /// Token is not valid to join the room. + SIGNAL_FAILURE_REASON_UNAUTHORIZED = 1 << 2, + + /// Other client error not covered by other reasons. + SIGNAL_FAILURE_REASON_CLIENT_OTHER = 1 << 3, + + /// Any client error, no retry should be attempted. + SIGNAL_FAILURE_REASON_CLIENT_ANY = SIGNAL_FAILURE_REASON_BAD_TOKEN | + SIGNAL_FAILURE_REASON_UNAUTHORIZED | + SIGNAL_FAILURE_REASON_CLIENT_OTHER, + /// Internal server error. + SIGNAL_FAILURE_REASON_INTERNAL = 1 << 4 +} signal_failure_reason_t; + typedef struct { void* ctx; @@ -46,6 +71,14 @@ signal_err_t signal_connect(signal_handle_t handle, const char* server_url, cons /// Closes the WebSocket connection signal_err_t signal_close(signal_handle_t handle); +/// Returns the reason why the connection failed. +/// +/// Use after the signal client's state changes to `CONNECTION_STATE_FAILED`. +/// Will be reset to `SIGNAL_FAILURE_REASON_NONE` during the next connection attempt. +/// +signal_failure_reason_t signal_get_failure_reason(signal_handle_t handle); + +/// Sends a leave request. signal_err_t signal_send_leave(signal_handle_t handle); signal_err_t signal_send_offer(signal_handle_t handle, const char *sdp); signal_err_t signal_send_answer(signal_handle_t handle, const char *sdp); diff --git a/components/livekit/include/livekit.h b/components/livekit/include/livekit.h index 1ed8f42..aa3b6f3 100644 --- a/components/livekit/include/livekit.h +++ b/components/livekit/include/livekit.h @@ -5,6 +5,7 @@ #include "esp_capture.h" #include "av_render.h" +#include "livekit_types.h" #include "livekit_rpc.h" #ifdef __cplusplus @@ -22,16 +23,6 @@ typedef enum { LIVEKIT_ERR_SYSTEM_INIT = -6 ///< System not initialized } livekit_err_t; -/// Connection state of a room. -/// @ingroup Connection -typedef enum { - LIVEKIT_CONNECTION_STATE_DISCONNECTED = 0, ///< Disconnected - LIVEKIT_CONNECTION_STATE_CONNECTING = 1, ///< Establishing connection - LIVEKIT_CONNECTION_STATE_CONNECTED = 2, ///< Connected - LIVEKIT_CONNECTION_STATE_RECONNECTING = 3, ///< Connection was previously established, but was lost - LIVEKIT_CONNECTION_STATE_FAILED = 4 ///< Connection failed -} livekit_connection_state_t; - /// Video codec to use within a room. typedef enum { LIVEKIT_VIDEO_CODEC_NONE = 0, ///< No video codec set @@ -295,6 +286,23 @@ livekit_connection_state_t livekit_room_get_state(livekit_room_handle_t handle); /// const char* livekit_connection_state_str(livekit_connection_state_t state); +/// Gets the reason why the room connection failed. +/// +/// Use this to check why the room connection failed after the room's state changes to +/// `LIVEKIT_CONNECTION_STATE_FAILED` or `LIVEKIT_CONNECTION_STATE_RECONNECTING`. +/// +/// @param handle[in] Room handle. +/// @return Failure reason. +/// +livekit_failure_reason_t livekit_room_get_failure_reason(livekit_room_handle_t handle); + +/// Gets a string representation for a failure reason. +/// +/// @param reason[in] Failure reason. +/// @return String representation of the failure reason. +/// +const char* livekit_failure_reason_str(livekit_failure_reason_t reason); + /// @} /// @defgroup Info Room & Participant Info diff --git a/components/livekit/include/livekit_types.h b/components/livekit/include/livekit_types.h new file mode 100644 index 0000000..95a574f --- /dev/null +++ b/components/livekit/include/livekit_types.h @@ -0,0 +1,67 @@ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// Connection state of a room. +/// @ingroup Connection +typedef enum { + LIVEKIT_CONNECTION_STATE_DISCONNECTED = 0, ///< Disconnected + LIVEKIT_CONNECTION_STATE_CONNECTING = 1, ///< Establishing connection + LIVEKIT_CONNECTION_STATE_CONNECTED = 2, ///< Connected + LIVEKIT_CONNECTION_STATE_RECONNECTING = 3, ///< Reestablishing connection after a failure + LIVEKIT_CONNECTION_STATE_FAILED = 4 ///< Connection failed after maximum number of retries +} livekit_connection_state_t; + +/// Reason why room connection failed. +/// @ingroup Connection +typedef enum { + /// No failure has occurred. + LIVEKIT_FAILURE_REASON_NONE, + + /// LiveKit server could not be reached. + /// + /// This may occur due to network connectivity issues, incorrect URL, + /// TLS handshake failure, or an offline server. + /// + LIVEKIT_FAILURE_REASON_UNREACHABLE, + + /// Token is malformed. + /// + /// This can occur if the token has missing/empty identity or room fields, + /// or if either of these fields exceeds the maximum length. + /// + LIVEKIT_FAILURE_REASON_BAD_TOKEN, + + /// Token is not valid to join the room. + /// + /// This can be caused by an expired token, or a token that lacks + /// necessary claims. + /// + LIVEKIT_FAILURE_REASON_UNAUTHORIZED, + + /// WebRTC establishment failure. + /// + /// Required peer connection(s) could not be established or failed. + /// + LIVEKIT_FAILURE_REASON_RTC, + + /// Maximum number of retries reached. + /// + /// Room connection failed after `CONFIG_LK_MAX_RETRIES` retries. + /// + LIVEKIT_FAILURE_REASON_MAX_RETRIES, + + /// Other failure reason. + /// + /// Any other failure not covered by other reasons. Check console output + /// for more details, and please report the issue on GitHub. + /// + LIVEKIT_FAILURE_REASON_OTHER +} livekit_failure_reason_t; + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/examples/minimal/main/example.c b/examples/minimal/main/example.c index e14eec7..f7a8f32 100644 --- a/examples/minimal/main/example.c +++ b/examples/minimal/main/example.c @@ -12,7 +12,11 @@ static livekit_room_handle_t room_handle; /// Invoked when the room's connection state changes. static void on_state_changed(livekit_connection_state_t state, void* ctx) { - ESP_LOGI(TAG, "Room state: %s", livekit_connection_state_str(state)); + ESP_LOGI(TAG, "Room state changed: %s", livekit_connection_state_str(state)); + livekit_failure_reason_t reason = livekit_room_get_failure_reason(room_handle); + if (reason != LIVEKIT_FAILURE_REASON_NONE) { + ESP_LOGE(TAG, "Failure reason: %s", livekit_failure_reason_str(reason)); + } } void join_room() From 517b3e5bae6e1d4f39af06ef5da6aa10f553a3db Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 21 Aug 2025 15:39:44 +1000 Subject: [PATCH 35/81] Extract into helpers --- components/livekit/core/engine.c | 138 ++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 48 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index b3dc98e..e18042c 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -555,6 +555,89 @@ static void flush_event_queue(engine_t *eng) } } +/// Starts the timer for the given period. +/// +/// Enqueues `EV_TIMER_EXP` after the period has elapsed. +/// +static inline void timer_start(engine_t *eng, uint16_t period) +{ + xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(period), 0); + xTimerStart(eng->timer, 0); +} + +/// Stops the timer. +static inline void timer_stop(engine_t *eng) +{ + xTimerStop(eng->timer, 0); +} + +static void handle_join(engine_t *eng, livekit_pb_join_response_t *join) +{ + // 1. Store connection settings + eng->is_subscriber_primary = join->subscriber_primary; + if (join->has_client_configuration) { + eng->force_relay = join->client_configuration.force_relay + == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; + } + + // 2. Store local Participant SID + strncpy( + eng->local_participant_sid, + join->participant.sid, + sizeof(eng->local_participant_sid) + ); + + // 3. Dispatch initial room info + if (eng->options.on_room_info && join->has_room) { + eng->options.on_room_info(&join->room, eng->options.ctx); + } + + // 4. Dispatch initial participant info + if (eng->options.on_participant_info) { + eng->options.on_participant_info(&join->participant, true, eng->options.ctx); + for (pb_size_t i = 0; i < join->other_participants_count; i++) { + eng->options.on_participant_info(&join->other_participants[i], false, eng->options.ctx); + } + } +} + +static void handle_trickle(engine_t *eng, livekit_pb_trickle_request_t *trickle) +{ + char* candidate = NULL; + if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { + return; + } + peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? + eng->pub_peer_handle : eng->sub_peer_handle; + peer_handle_ice_candidate(target_peer, candidate); + free(candidate); +} + +static void handle_room_update(engine_t *eng, livekit_pb_room_update_t *room_update) +{ + if (eng->options.on_room_info && room_update->has_room) { + eng->options.on_room_info(&room_update->room, eng->options.ctx); + } +} + +static void handle_participant_update(engine_t *eng, livekit_pb_participant_update_t *update) +{ + if (!eng->options.on_participant_info) { + return; + } + bool found_local = false; + for (pb_size_t i = 0; i < update->participants_count; i++) { + livekit_pb_participant_info_t *participant = &update->participants[i]; + bool is_local = !found_local && strncmp( + participant->sid, + eng->local_participant_sid, + sizeof(eng->local_participant_sid) + ) == 0; + if (is_local) found_local = true; + eng->options.on_participant_info(participant, is_local, eng->options.ctx); + } +} + // MARK: - FSM state handlers /// Handler for `ENGINE_STATE_DISCONNECTED`. @@ -609,18 +692,7 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: livekit_pb_join_response_t *join = &res->message.join; - // Store connection settings - eng->is_subscriber_primary = join->subscriber_primary; - if (join->has_client_configuration) { - eng->force_relay = join->client_configuration.force_relay - == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; - } - // Store local participant SID - strncpy( - eng->local_participant_sid, - join->participant.sid, - sizeof(eng->local_participant_sid) - ); + handle_join(eng, join); if (!establish_peer_connections(eng)) { ESP_LOGE(TAG, "Failed to establish peer connections"); eng->state = ENGINE_STATE_DISCONNECTED; @@ -637,14 +709,7 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) break; case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: livekit_pb_trickle_request_t *trickle = &res->message.trickle; - char* candidate = NULL; - if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { - break; - } - peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? - eng->pub_peer_handle : eng->sub_peer_handle; - peer_handle_ice_candidate(target_peer, candidate); - free(candidate); + handle_trickle(eng, trickle); break; default: break; @@ -714,26 +779,11 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) break; case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: livekit_pb_room_update_t *room_update = &res->message.room_update; - if (eng->options.on_room_info && room_update->has_room) { - eng->options.on_room_info(&room_update->room, eng->options.ctx); - } + handle_room_update(eng, room_update); break; case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: livekit_pb_participant_update_t *update = &res->message.update; - if (!eng->options.on_participant_info) { - break; - } - bool found_local = false; - for (pb_size_t i = 0; i < update->participants_count; i++) { - livekit_pb_participant_info_t *participant = &update->participants[i]; - bool is_local = !found_local && strncmp( - participant->sid, - eng->local_participant_sid, - sizeof(eng->local_participant_sid) - ) == 0; - if (is_local) found_local = true; - eng->options.on_participant_info(participant, is_local, eng->options.ctx); - } + handle_participant_update(eng, update); break; // TODO: Only handle if needed case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: @@ -746,14 +796,7 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) break; case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: livekit_pb_trickle_request_t *trickle = &res->message.trickle; - char* candidate = NULL; - if (!protocol_signal_trickle_get_candidate(trickle, &candidate)) { - break; - } - peer_handle_t target_peer = trickle->target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? - eng->pub_peer_handle : eng->sub_peer_handle; - peer_handle_ice_candidate(target_peer, candidate); - free(candidate); + handle_trickle(eng, trickle); break; default: break; @@ -802,8 +845,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) ESP_LOGI(TAG, "Reconnect in %" PRIu16 "ms: attempt=%d/%d, reason=%d", backoff_ms, eng->retry_count, CONFIG_LK_MAX_RETRIES, eng->failure_reason); - xTimerChangePeriod(eng->timer, pdMS_TO_TICKS(backoff_ms), 0); - xTimerStart(eng->timer, 0); + timer_start(eng, backoff_ms); break; case EV_MAX_RETRIES_REACHED: eng->failure_reason = LIVEKIT_FAILURE_REASON_MAX_RETRIES; @@ -813,7 +855,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_CONNECTING; break; case _EV_STATE_EXIT: - xTimerStop(eng->timer, portMAX_DELAY); + timer_stop(eng); break; default: break; From b2c9d2f46618b1ab5625de302585acaaf732fd1e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 21 Aug 2025 16:17:43 +1000 Subject: [PATCH 36/81] Unify peer state change handling --- components/livekit/core/engine.c | 119 +++++++++++++++---------------- components/livekit/core/peer.c | 32 ++++----- components/livekit/core/peer.h | 11 ++- 3 files changed, 79 insertions(+), 83 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index e18042c..a9d3ea7 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -32,8 +32,7 @@ typedef enum { EV_CMD_CLOSE, /// User-initiated disconnect. EV_SIG_STATE, /// Signal state changed. EV_SIG_RES, /// Signal response received. - EV_PEER_PUB_STATE, /// Publisher peer state changed. - EV_PEER_SUB_STATE, /// Subscriber peer state changed. + EV_PEER_STATE, /// Peer state changed. EV_PEER_DATA_PACKET, /// Peer received data packet. EV_TIMER_EXP, /// Timer expired. EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. @@ -58,8 +57,14 @@ typedef struct { /// Detail for `EV_PEER_DATA_PACKET`. livekit_pb_data_packet_t data_packet; - /// Detail for `EV_SIG_STATE`, `EV_PEER_PUB_STATE` and `EV_PEER_SUB_STATE`. - connection_state_t state; + /// Detail for `EV_SIG_STATE`. + connection_state_t sig_state; + + /// Detail for `EV_PEER_STATE`. + struct { + connection_state_t state; + peer_role_t role; + } peer_state; } detail; } engine_event_t; @@ -304,7 +309,7 @@ static void on_signal_state_changed(connection_state_t state, void *ctx) engine_t *eng = (engine_t *)ctx; engine_event_t ev = { .type = EV_SIG_STATE, - .detail.state = state + .detail.sig_state = state }; event_enqueue(eng, &ev, true); } @@ -324,6 +329,16 @@ static bool on_signal_res(livekit_pb_signal_response_t *res, void *ctx) // MARK: - Common peer event handlers +static void on_peer_state_changed(connection_state_t state, peer_role_t role, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + engine_event_t ev = { + .type = EV_PEER_STATE, + .detail.peer_state = { .state = state, .role = role } + }; + event_enqueue(eng, &ev, true); +} + static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) { engine_t *eng = (engine_t *)ctx; @@ -343,16 +358,6 @@ static void on_peer_ice_candidate(const char *candidate, void *ctx) // MARK: - Publisher peer event handlers -static void on_peer_pub_state_changed(connection_state_t state, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - engine_event_t ev = { - .type = EV_PEER_PUB_STATE, - .detail.state = state - }; - event_enqueue(eng, &ev, true); -} - static void on_peer_pub_offer(const char *sdp, void *ctx) { engine_t *eng = (engine_t *)ctx; @@ -361,16 +366,6 @@ static void on_peer_pub_offer(const char *sdp, void *ctx) // MARK: - Subscriber peer event handlers -static void on_peer_sub_state_changed(connection_state_t state, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - engine_event_t ev = { - .type = EV_PEER_SUB_STATE, - .detail.state = state - }; - event_enqueue(eng, &ev, true); -} - static void on_peer_sub_answer(const char *sdp, void *ctx) { engine_t *eng = (engine_t *)ctx; @@ -444,28 +439,27 @@ static void destroy_peer_connections(engine_t *eng) static bool establish_peer_connections(engine_t *eng) { peer_options_t options = { - .force_relay = eng->force_relay, - .media = &eng->options.media, - .on_data_packet = on_peer_data_packet, - .on_ice_candidate = on_peer_ice_candidate, - .ctx = eng + .force_relay = eng->force_relay, + .media = &eng->options.media, + .on_state_changed = on_peer_state_changed, + .on_data_packet = on_peer_data_packet, + .on_ice_candidate = on_peer_ice_candidate, + .ctx = eng }; - // Publisher - options.target = LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER; - options.on_state_changed = on_peer_pub_state_changed; - options.on_sdp = on_peer_pub_offer; + // 1. Publisher + options.role = PEER_ROLE_PUBLISHER; + options.on_sdp = on_peer_pub_offer; _create_and_connect_peer(&options, &eng->pub_peer_handle); if (eng->pub_peer_handle == NULL) return false; - // Subscriber - options.target = LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER; - options.on_state_changed = on_peer_sub_state_changed; - options.on_sdp = on_peer_sub_answer; - options.on_audio_info = on_peer_sub_audio_info; - options.on_audio_frame = on_peer_sub_audio_frame; + // 2. Subscriber + options.role = PEER_ROLE_SUBSCRIBER; + options.on_sdp = on_peer_sub_answer; + options.on_audio_info = on_peer_sub_audio_info; + options.on_audio_frame = on_peer_sub_audio_frame; _create_and_connect_peer(&options, &eng->sub_peer_handle); if (eng->sub_peer_handle == NULL) { @@ -497,8 +491,7 @@ static inline livekit_connection_state_t map_engine_state(engine_t *eng) return LIVEKIT_CONNECTION_STATE_RECONNECTING; case ENGINE_STATE_CONNECTED: return LIVEKIT_CONNECTION_STATE_CONNECTED; - default: - return LIVEKIT_CONNECTION_STATE_DISCONNECTED; + default: return -1; } } @@ -716,7 +709,7 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED) { + if (ev->detail.sig_state == CONNECTION_STATE_FAILED) { signal_failure_reason_t reason = signal_get_failure_reason(eng->signal_handle); eng->failure_reason = map_signal_failure_reason(reason); // Client errors should not trigger a reconnection @@ -725,20 +718,22 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) ENGINE_STATE_BACKOFF; } break; - case EV_PEER_PUB_STATE: - if (!eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { - eng->state = ENGINE_STATE_CONNECTED; - } else if (ev->detail.state == CONNECTION_STATE_FAILED) { + case EV_PEER_STATE: + connection_state_t state = ev->detail.peer_state.state; + peer_role_t role = ev->detail.peer_state.role; + + // If either peer fails, transition to backoff + if (state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; + break; } - break; - case EV_PEER_SUB_STATE: - if (eng->is_subscriber_primary && ev->detail.state == CONNECTION_STATE_CONNECTED) { - eng->state = ENGINE_STATE_CONNECTED; - } else if (ev->detail.state == CONNECTION_STATE_FAILED) { - eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; - eng->state = ENGINE_STATE_BACKOFF; + // Once the primary peer is connected, transition to connected + if (state == CONNECTION_STATE_CONNECTED) { + if ((role == PEER_ROLE_PUBLISHER && !eng->is_subscriber_primary) || + (role == PEER_ROLE_SUBSCRIBER && eng->is_subscriber_primary)) { + eng->state = ENGINE_STATE_CONNECTED; + } } break; default: @@ -803,19 +798,17 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED) { + if (ev->detail.sig_state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_UNREACHABLE; eng->state = ENGINE_STATE_BACKOFF; } break; - case EV_PEER_PUB_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED) { - eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; - eng->state = ENGINE_STATE_BACKOFF; - } - break; - case EV_PEER_SUB_STATE: - if (ev->detail.state == CONNECTION_STATE_FAILED) { + case EV_PEER_STATE: + connection_state_t state = ev->detail.peer_state.state; + peer_role_t role = ev->detail.peer_state.role; + + // If either peer fails, transition to backoff + if (state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index edcc675..1241a74 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -11,7 +11,7 @@ static const char *SUB_TAG = "livekit_peer.sub"; static const char *PUB_TAG = "livekit_peer.pub"; -#define TAG(peer) (peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? SUB_TAG : PUB_TAG) +#define TAG(peer) (peer->options.role == PEER_ROLE_SUBSCRIBER ? SUB_TAG : PUB_TAG) #define SUB_THREAD_NAME (PEER_THREAD_NAME_PREFIX "sub") #define PUB_THREAD_NAME (PEER_THREAD_NAME_PREFIX "pub") @@ -40,14 +40,12 @@ typedef struct { uint16_t lossy_stream_id; } peer_t; -static esp_peer_media_dir_t get_media_direction(esp_peer_media_dir_t direction, livekit_pb_signal_target_t target) { - switch (target) { - case LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER: - return direction & ESP_PEER_MEDIA_DIR_SEND_ONLY; - case LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER: - return direction & ESP_PEER_MEDIA_DIR_RECV_ONLY; +static esp_peer_media_dir_t get_media_direction(esp_peer_media_dir_t direction, peer_role_t role) { + switch (role) { + case PEER_ROLE_PUBLISHER: return direction & ESP_PEER_MEDIA_DIR_SEND_ONLY; + case PEER_ROLE_SUBSCRIBER: return direction & ESP_PEER_MEDIA_DIR_RECV_ONLY; + default: return ESP_PEER_MEDIA_DIR_NONE; } - return ESP_PEER_MEDIA_DIR_NONE; } static void peer_task(void *ctx) @@ -69,7 +67,7 @@ static void peer_task(void *ctx) static void create_data_channels(peer_t *peer) { - if (peer->options.target != LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER) return; + if (peer->options.role != PEER_ROLE_PUBLISHER) return; esp_peer_data_channel_cfg_t reliable_cfg = { .label = RELIABLE_CHANNEL_LABEL, @@ -122,7 +120,7 @@ static int on_state(esp_peer_state_t rtc_state, void *ctx) if (new_state != peer->state) { ESP_LOGI(TAG(peer), "State changed: %d -> %d", peer->state, new_state); peer->state = new_state; - peer->options.on_state_changed(new_state, peer->options.ctx); + peer->options.on_state_changed(new_state, peer->options.role, peer->options.ctx); } return 0; } @@ -133,7 +131,7 @@ static int on_msg(esp_peer_msg_t *info, void *ctx) switch (info->type) { case ESP_PEER_MSG_TYPE_SDP: ESP_LOGI(TAG(peer), "Generated %s:\n%s", - peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER ? "offer" : "answer", + peer->options.role == PEER_ROLE_PUBLISHER ? "offer" : "answer", (char *)info->data); peer->options.on_sdp((char *)info->data, peer->options.ctx); break; @@ -260,7 +258,7 @@ peer_err_t peer_create(peer_handle_t *handle, peer_options_t *options) } peer->options = *options; - peer->ice_role = options->target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? + peer->ice_role = options->role == PEER_ROLE_SUBSCRIBER ? ESP_PEER_ROLE_CONTROLLED : ESP_PEER_ROLE_CONTROLLING; peer->state = CONNECTION_STATE_DISCONNECTED; @@ -278,8 +276,8 @@ peer_err_t peer_create(peer_handle_t *handle, peer_options_t *options) } // TODO: Set options }; - esp_peer_media_dir_t audio_dir = get_media_direction(options->media->audio_dir, peer->options.target); - esp_peer_media_dir_t video_dir = get_media_direction(options->media->video_dir, peer->options.target); + esp_peer_media_dir_t audio_dir = get_media_direction(options->media->audio_dir, peer->options.role); + esp_peer_media_dir_t video_dir = get_media_direction(options->media->video_dir, peer->options.role); ESP_LOGD(TAG(peer), "Audio dir: %d, Video dir: %d", audio_dir, video_dir); esp_peer_cfg_t peer_cfg = { @@ -337,7 +335,7 @@ peer_err_t peer_connect(peer_handle_t handle) peer->running = true; media_lib_thread_handle_t thread; - const char* thread_name = peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? + const char* thread_name = peer->options.role == PEER_ROLE_SUBSCRIBER ? SUB_THREAD_NAME : PUB_THREAD_NAME; if (media_lib_thread_create_from_scheduler(&thread, thread_name, peer_task, peer) != ESP_PEER_ERR_NONE) { ESP_LOGE(TAG(peer), "Failed to create thread"); @@ -474,7 +472,7 @@ peer_err_t peer_send_audio(peer_handle_t handle, esp_peer_audio_frame_t* frame) return PEER_ERR_INVALID_ARG; } peer_t *peer = (peer_t *)handle; - assert(peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER); + assert(peer->options.role == PEER_ROLE_PUBLISHER); esp_peer_send_audio(peer->connection, frame); return PEER_ERR_NONE; @@ -486,7 +484,7 @@ peer_err_t peer_send_video(peer_handle_t handle, esp_peer_video_frame_t* frame) return PEER_ERR_INVALID_ARG; } peer_t *peer = (peer_t *)handle; - assert(peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_PUBLISHER); + assert(peer->options.role == PEER_ROLE_PUBLISHER); esp_peer_send_video(peer->connection, frame); return PEER_ERR_NONE; diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index 9f6d41f..23c4c70 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -21,10 +21,15 @@ typedef enum { PEER_ERR_MESSAGE = -5 } peer_err_t; +typedef enum { + PEER_ROLE_PUBLISHER, + PEER_ROLE_SUBSCRIBER +} peer_role_t; + /// Options for creating a peer. typedef struct { - /// Whether the peer is a publisher or subscriber. - livekit_pb_signal_target_t target; + /// Peer role (publisher or subscriber). + peer_role_t role; /// ICE server list. esp_peer_ice_server_cfg_t* server_list; @@ -39,7 +44,7 @@ typedef struct { engine_media_options_t* media; /// Invoked when the peer's connection state changes. - void (*on_state_changed)(connection_state_t state, void *ctx); + void (*on_state_changed)(connection_state_t state, peer_role_t role, void *ctx); /// Invoked when a data packet is received over the data channel. /// From 197d7ad434521c6a9dfb2fb49ad6331df3e7f390 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 21 Aug 2025 16:34:06 +1000 Subject: [PATCH 37/81] Free event if cannot enqueue --- components/livekit/core/engine.c | 1 + 1 file changed, 1 insertion(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index a9d3ea7..f89d05c 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1034,6 +1034,7 @@ engine_err_t engine_connect(engine_handle_t handle, const char* server_url, cons .detail.cmd_connect = { .server_url = strdup(server_url), .token = strdup(token) } }; if (!event_enqueue(eng, &ev, true)) { + event_free(&ev); return ENGINE_ERR_OTHER; } return ENGINE_ERR_NONE; From 985b89c12b77ca950606e4a91384be0e460e20b0 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 11:30:31 +1000 Subject: [PATCH 38/81] URL build options struct --- components/livekit/core/signaling.c | 8 ++++++-- components/livekit/core/url.c | 20 ++++++++++++-------- components/livekit/core/url.h | 14 +++++++++++--- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 37d5aa1..07cfd0b 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -249,8 +249,12 @@ signal_err_t signal_connect(signal_handle_t handle, const char* server_url, cons } signal_t *sg = (signal_t *)handle; - char* url; - if (!url_build(server_url, token, &url)) { + char* url = NULL; + url_build_options options = { + .server_url = server_url, + .token = token + }; + if (!url_build(&options, &url)) { return SIGNAL_ERR_INVALID_URL; } esp_websocket_client_set_uri(sg->ws, url); diff --git a/components/livekit/core/url.c b/components/livekit/core/url.c index e749075..7443829 100644 --- a/components/livekit/core/url.c +++ b/components/livekit/core/url.c @@ -26,22 +26,26 @@ static const char *TAG = "livekit_url"; "&protocol=" URL_PARAM_PROTOCOL \ "&access_token=%s" // Keep at the end for log redaction -bool url_build(const char *server_url, const char *token, char **out_url) +bool url_build(const url_build_options *options, char **out_url) { - if (server_url == NULL || token == NULL || out_url == NULL) { + if (out_url == NULL || + options == NULL || + options->server_url == NULL || + options->token == NULL) { return false; } - size_t server_url_len = strlen(server_url); + size_t server_url_len = strlen(options->server_url); if (server_url_len < 1) { ESP_LOGE(TAG, "Server URL cannot be empty"); return false; } - if (strncmp(server_url, "ws://", 5) != 0 && strncmp(server_url, "wss://", 6) != 0) { + if (strncmp(options->server_url, "ws://", 5) != 0 && + strncmp(options->server_url, "wss://", 6) != 0) { ESP_LOGE(TAG, "Unsupported URL scheme"); return false; } // Do not add a trailing slash if the URL already has one - const char *separator = server_url[server_url_len - 1] == '/' ? "" : "/"; + const char *separator = options->server_url[server_url_len - 1] == '/' ? "" : "/"; // Get chip and OS information esp_chip_info_t chip_info; @@ -50,18 +54,18 @@ bool url_build(const char *server_url, const char *token, char **out_url) const char* idf_version = esp_get_idf_version(); int final_len = asprintf(out_url, URL_FORMAT, - server_url, + options->server_url, separator, idf_version, model_code, - token + options->token ); if (*out_url == NULL) { return false; } // Token is redacted from logging for security ESP_LOGI(TAG, "Built signaling URL: %.*s[REDACTED]", - (int)((size_t)final_len - strlen(token)), + (int)((size_t)final_len - strlen(options->token)), *out_url); return true; } \ No newline at end of file diff --git a/components/livekit/core/url.h b/components/livekit/core/url.h index 43630c7..e27f5eb 100644 --- a/components/livekit/core/url.h +++ b/components/livekit/core/url.h @@ -7,13 +7,21 @@ extern "C" { #endif +/// Options for building a signaling URL. +typedef struct { + const char *server_url; + const char *token; +} url_build_options; + /// Constructs a signaling URL. -/// @param server_url The server URL beginning with ws:// or wss://. -/// @param token Access token. +/// +/// @param options The options for building the URL. /// @param out_url[out] The output URL. +/// /// @return True if the URL is constructed successfully, false otherwise. /// @note The caller is responsible for freeing the output URL. -bool url_build(const char *server_url, const char *token, char **out_url); +/// +bool url_build(const url_build_options *options, char **out_url); #ifdef __cplusplus } From 6a9cbee6b129416a622a5d4c41cc0dd3144105bd Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 11:46:08 +1000 Subject: [PATCH 39/81] Map disconnect reason to failure reason --- components/livekit/core/engine.c | 32 ++++- components/livekit/core/livekit.c | 28 +++-- components/livekit/include/livekit_types.h | 140 +++++++++++++++++++-- 3 files changed, 176 insertions(+), 24 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index f89d05c..8822398 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -495,7 +495,7 @@ static inline livekit_connection_state_t map_engine_state(engine_t *eng) } } -/// Map a signal failure reason to the failure reason exposed in the public room API. +/// Map a signal failure reason to a failure reason exposed in the public room API. static livekit_failure_reason_t map_signal_failure_reason(signal_failure_reason_t reason) { switch (reason) { @@ -506,6 +506,29 @@ static livekit_failure_reason_t map_signal_failure_reason(signal_failure_reason_ } } +/// Map a protocol disconnect reason to a failure reason exposed in the public room API. +static livekit_failure_reason_t map_disconnect_reason(livekit_pb_disconnect_reason_t reason) +{ + switch (reason) { + case LIVEKIT_PB_DISCONNECT_REASON_CLIENT_INITIATED: return LIVEKIT_FAILURE_REASON_NONE; + case LIVEKIT_PB_DISCONNECT_REASON_DUPLICATE_IDENTITY: return LIVEKIT_FAILURE_REASON_DUPLICATE_IDENTITY; + case LIVEKIT_PB_DISCONNECT_REASON_SERVER_SHUTDOWN: return LIVEKIT_FAILURE_REASON_SERVER_SHUTDOWN; + case LIVEKIT_PB_DISCONNECT_REASON_PARTICIPANT_REMOVED: return LIVEKIT_FAILURE_REASON_PARTICIPANT_REMOVED; + case LIVEKIT_PB_DISCONNECT_REASON_ROOM_DELETED: return LIVEKIT_FAILURE_REASON_ROOM_DELETED; + case LIVEKIT_PB_DISCONNECT_REASON_STATE_MISMATCH: return LIVEKIT_FAILURE_REASON_STATE_MISMATCH; + case LIVEKIT_PB_DISCONNECT_REASON_JOIN_FAILURE: return LIVEKIT_FAILURE_REASON_JOIN_INCOMPLETE; + case LIVEKIT_PB_DISCONNECT_REASON_MIGRATION: return LIVEKIT_FAILURE_REASON_MIGRATION; + case LIVEKIT_PB_DISCONNECT_REASON_SIGNAL_CLOSE: return LIVEKIT_FAILURE_REASON_SIGNAL_CLOSE; + case LIVEKIT_PB_DISCONNECT_REASON_ROOM_CLOSED: return LIVEKIT_FAILURE_REASON_ROOM_CLOSED; + case LIVEKIT_PB_DISCONNECT_REASON_USER_UNAVAILABLE: return LIVEKIT_FAILURE_REASON_SIP_USER_UNAVAILABLE; + case LIVEKIT_PB_DISCONNECT_REASON_USER_REJECTED: return LIVEKIT_FAILURE_REASON_SIP_USER_REJECTED; + case LIVEKIT_PB_DISCONNECT_REASON_SIP_TRUNK_FAILURE: return LIVEKIT_FAILURE_REASON_SIP_TRUNK_FAILURE; + case LIVEKIT_PB_DISCONNECT_REASON_CONNECTION_TIMEOUT: return LIVEKIT_FAILURE_REASON_CONNECTION_TIMEOUT; + case LIVEKIT_PB_DISCONNECT_REASON_MEDIA_FAILURE: return LIVEKIT_FAILURE_REASON_MEDIA_FAILURE; + default: return LIVEKIT_FAILURE_REASON_OTHER; + } +} + /// Frees an event's dynamically allocated fields (if any). static void event_free(engine_event_t *ev) { @@ -680,7 +703,8 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) livekit_pb_signal_response_t *res = &ev->detail.res; switch (res->which_message) { case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: - ESP_LOGI(TAG, "Server sent leave before fully connected"); + livekit_pb_leave_request_t *leave = &res->message.leave; + eng->failure_reason = map_disconnect_reason(leave->reason); eng->state = ENGINE_STATE_DISCONNECTED; break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: @@ -768,8 +792,8 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) livekit_pb_signal_response_t *res = &ev->detail.res; switch (ev->detail.res.which_message) { case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: - ESP_LOGI(TAG, "Server initiated disconnect"); - // TODO: Handle leave action + livekit_pb_leave_request_t *leave = &res->message.leave; + eng->failure_reason = map_disconnect_reason(leave->reason); eng->state = ENGINE_STATE_DISCONNECTED; break; case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index 01b1517..b47a14f 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -298,13 +298,27 @@ const char* livekit_connection_state_str(livekit_connection_state_t state) const char* livekit_failure_reason_str(livekit_failure_reason_t reason) { switch (reason) { - case LIVEKIT_FAILURE_REASON_NONE: return "None"; - case LIVEKIT_FAILURE_REASON_UNREACHABLE: return "Unreachable"; - case LIVEKIT_FAILURE_REASON_BAD_TOKEN: return "Bad Token"; - case LIVEKIT_FAILURE_REASON_UNAUTHORIZED: return "Unauthorized"; - case LIVEKIT_FAILURE_REASON_RTC: return "RTC"; - case LIVEKIT_FAILURE_REASON_MAX_RETRIES: return "Max Retries"; - default: return "Other"; + case LIVEKIT_FAILURE_REASON_NONE: return "None"; + case LIVEKIT_FAILURE_REASON_UNREACHABLE: return "Unreachable"; + case LIVEKIT_FAILURE_REASON_BAD_TOKEN: return "Bad Token"; + case LIVEKIT_FAILURE_REASON_UNAUTHORIZED: return "Unauthorized"; + case LIVEKIT_FAILURE_REASON_RTC: return "RTC"; + case LIVEKIT_FAILURE_REASON_MAX_RETRIES: return "Max Retries"; + case LIVEKIT_FAILURE_REASON_DUPLICATE_IDENTITY: return "Duplicate Identity"; + case LIVEKIT_FAILURE_REASON_SERVER_SHUTDOWN: return "Server Shutdown"; + case LIVEKIT_FAILURE_REASON_PARTICIPANT_REMOVED: return "Participant Removed"; + case LIVEKIT_FAILURE_REASON_ROOM_DELETED: return "Room Deleted"; + case LIVEKIT_FAILURE_REASON_STATE_MISMATCH: return "State Mismatch"; + case LIVEKIT_FAILURE_REASON_JOIN_INCOMPLETE: return "Join Incomplete"; + case LIVEKIT_FAILURE_REASON_MIGRATION: return "Migration"; + case LIVEKIT_FAILURE_REASON_SIGNAL_CLOSE: return "Signal Close"; + case LIVEKIT_FAILURE_REASON_ROOM_CLOSED: return "Room Closed"; + case LIVEKIT_FAILURE_REASON_SIP_USER_UNAVAILABLE: return "SIP User Unavailable"; + case LIVEKIT_FAILURE_REASON_SIP_USER_REJECTED: return "SIP User Rejected"; + case LIVEKIT_FAILURE_REASON_SIP_TRUNK_FAILURE: return "SIP Trunk Failure"; + case LIVEKIT_FAILURE_REASON_CONNECTION_TIMEOUT: return "Connection Timeout"; + case LIVEKIT_FAILURE_REASON_MEDIA_FAILURE: return "Media Failure"; + default: return "Other"; } } diff --git a/components/livekit/include/livekit_types.h b/components/livekit/include/livekit_types.h index 95a574f..6650ae6 100644 --- a/components/livekit/include/livekit_types.h +++ b/components/livekit/include/livekit_types.h @@ -21,39 +21,153 @@ typedef enum { /// No failure has occurred. LIVEKIT_FAILURE_REASON_NONE, - /// LiveKit server could not be reached. + /// Unreachable /// - /// This may occur due to network connectivity issues, incorrect URL, - /// TLS handshake failure, or an offline server. + /// LiveKit server could not be reached: this may occur due to network + /// connectivity issues, incorrect URL, TLS handshake failure, or an offline server. /// LIVEKIT_FAILURE_REASON_UNREACHABLE, - /// Token is malformed. + /// Bad Token /// - /// This can occur if the token has missing/empty identity or room fields, - /// or if either of these fields exceeds the maximum length. + /// Token is malformed: this can occur if the token has missing/empty identity + /// or room fields, or if either of these fields exceeds the maximum length. /// LIVEKIT_FAILURE_REASON_BAD_TOKEN, - /// Token is not valid to join the room. + /// Unauthorized /// - /// This can be caused by an expired token, or a token that lacks - /// necessary claims. + /// Token is not valid to join the room: this can be caused by an + /// expired token, or a token that lacks necessary claims. /// LIVEKIT_FAILURE_REASON_UNAUTHORIZED, - /// WebRTC establishment failure. + /// RTC /// - /// Required peer connection(s) could not be established or failed. + /// WebRTC establishment failure: required peer connection(s) could + /// not be established or failed. /// LIVEKIT_FAILURE_REASON_RTC, - /// Maximum number of retries reached. + /// Max Retries /// - /// Room connection failed after `CONFIG_LK_MAX_RETRIES` retries. + /// Maximum number of retries reached: room connection failed after + /// `CONFIG_LK_MAX_RETRIES` retries. /// LIVEKIT_FAILURE_REASON_MAX_RETRIES, + /// Duplicate Identity + /// + /// Another participant already has the same identity. + /// + /// Protocol equivalent: `DisconnectReason.DUPLICATE_IDENTITY`. + /// + LIVEKIT_FAILURE_REASON_DUPLICATE_IDENTITY, + + /// Server Shutdown + /// + /// LiveKit server instance is shutting down. + /// + /// Protocol equivalent: `DisconnectReason.SERVER_SHUTDOWN`. + /// + LIVEKIT_FAILURE_REASON_SERVER_SHUTDOWN, + + /// Participant Removed + /// + /// Participant was removed using room services API. + /// + /// Protocol equivalent: `DisconnectReason.PARTICIPANT_REMOVED`. + /// + LIVEKIT_FAILURE_REASON_PARTICIPANT_REMOVED, + + /// Room Deleted + /// + /// Room was deleted using room services API. + /// + /// Protocol equivalent: `DisconnectReason.ROOM_DELETED`. + /// + LIVEKIT_FAILURE_REASON_ROOM_DELETED, + + /// State Mismatch + /// + /// Client attempted to resume, but server is not aware of it. + /// + /// Protocol equivalent: `DisconnectReason.STATE_MISMATCH`. + /// + LIVEKIT_FAILURE_REASON_STATE_MISMATCH, + + /// Join Incomplete + /// + /// Client was unable to fully establish a connection. + /// + /// Protocol equivalent: `DisconnectReason.JOIN_FAILURE`. + /// + LIVEKIT_FAILURE_REASON_JOIN_INCOMPLETE, + + /// Migration + /// + /// The server requested the client to migrate the connection elsewhere (cloud only). + /// + /// Protocol equivalent: `DisconnectReason.MIGRATION`. + /// + LIVEKIT_FAILURE_REASON_MIGRATION, + + /// Signal Close + /// + /// The signal connection was closed unexpectedly. + /// + /// Protocol equivalent: `DisconnectReason.SIGNAL_CLOSE`. + /// + LIVEKIT_FAILURE_REASON_SIGNAL_CLOSE, + + /// Room Closed + /// + /// The room was closed, due to all Standard and Ingress participants having left. + /// + /// Protocol equivalent: `DisconnectReason.ROOM_CLOSED`. + /// + LIVEKIT_FAILURE_REASON_ROOM_CLOSED, + + /// SIP User Unavailable + /// + /// SIP callee did not respond in time. + /// + /// Protocol equivalent: `DisconnectReason.USER_UNAVAILABLE`. + /// + LIVEKIT_FAILURE_REASON_SIP_USER_UNAVAILABLE, + + /// SIP User Rejected + /// + /// SIP callee rejected the call (busy). + /// + /// Protocol equivalent: `DisconnectReason.USER_REJECTED`. + /// + LIVEKIT_FAILURE_REASON_SIP_USER_REJECTED, + + /// SIP Trunk Failure + /// + /// SIP protocol failure or unexpected response. + /// + /// Protocol equivalent: `DisconnectReason.SIP_TRUNK_FAILURE`. + /// + LIVEKIT_FAILURE_REASON_SIP_TRUNK_FAILURE, + + /// Connection Timeout + /// + /// Server timed out a participant session. + /// + /// Protocol equivalent: `DisconnectReason.CONNECTION_TIMEOUT`. + /// + LIVEKIT_FAILURE_REASON_CONNECTION_TIMEOUT, + + /// Media Failure + /// + /// Media stream failure or media timeout. + /// + /// Protocol equivalent: `DisconnectReason.MEDIA_FAILURE`. + /// + LIVEKIT_FAILURE_REASON_MEDIA_FAILURE, + /// Other failure reason. /// /// Any other failure not covered by other reasons. Check console output From 3292e8be9ca142fde6b15f6d0367df7a7723b136 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 11:46:28 +1000 Subject: [PATCH 40/81] Send leave on user-initiated close --- components/livekit/core/engine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 8822398..342746b 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -693,7 +693,7 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) signal_connect(eng->signal_handle, eng->server_url, eng->token); break; case EV_CMD_CLOSE: - // TODO: Send leave request + signal_send_leave(eng->signal_handle); eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_CMD_CONNECT: @@ -776,7 +776,7 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) publish_tracks(eng); break; case EV_CMD_CLOSE: - // TODO: Send leave request + signal_send_leave(eng->signal_handle); eng->state = ENGINE_STATE_DISCONNECTED; break; case EV_CMD_CONNECT: From f100bf66e0c628b710bae0f8ee4f4eb49491acdc Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 11:55:15 +1000 Subject: [PATCH 41/81] Separate session state --- components/livekit/core/engine.c | 33 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 342746b..76bde72 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -68,6 +68,12 @@ typedef struct { } detail; } engine_event_t; +typedef struct { + bool is_subscriber_primary; + bool force_relay; + livekit_pb_sid_t local_participant_sid; +} session_state_t; + typedef struct { engine_state_t state; engine_options_t options; @@ -80,12 +86,9 @@ typedef struct { esp_capture_path_handle_t capturer_path; bool is_media_streaming; - // Session state - bool is_subscriber_primary; - bool force_relay; char* server_url; char* token; - livekit_pb_sid_t local_participant_sid; + session_state_t session; TaskHandle_t task_handle; QueueHandle_t event_queue; @@ -439,7 +442,7 @@ static void destroy_peer_connections(engine_t *eng) static bool establish_peer_connections(engine_t *eng) { peer_options_t options = { - .force_relay = eng->force_relay, + .force_relay = eng->session.force_relay, .media = &eng->options.media, .on_state_changed = on_peer_state_changed, .on_data_packet = on_peer_data_packet, @@ -590,17 +593,17 @@ static inline void timer_stop(engine_t *eng) static void handle_join(engine_t *eng, livekit_pb_join_response_t *join) { // 1. Store connection settings - eng->is_subscriber_primary = join->subscriber_primary; + eng->session.is_subscriber_primary = join->subscriber_primary; if (join->has_client_configuration) { - eng->force_relay = join->client_configuration.force_relay + eng->session.force_relay = join->client_configuration.force_relay == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; } // 2. Store local Participant SID strncpy( - eng->local_participant_sid, + eng->session.local_participant_sid, join->participant.sid, - sizeof(eng->local_participant_sid) + sizeof(eng->session.local_participant_sid) ); // 3. Dispatch initial room info @@ -646,8 +649,8 @@ static void handle_participant_update(engine_t *eng, livekit_pb_participant_upda livekit_pb_participant_info_t *participant = &update->participants[i]; bool is_local = !found_local && strncmp( participant->sid, - eng->local_participant_sid, - sizeof(eng->local_participant_sid) + eng->session.local_participant_sid, + sizeof(eng->session.local_participant_sid) ) == 0; if (is_local) found_local = true; eng->options.on_participant_info(participant, is_local, eng->options.ctx); @@ -666,9 +669,7 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) signal_close(eng->signal_handle); destroy_peer_connections(eng); - eng->is_subscriber_primary = false; - eng->force_relay = false; - eng->local_participant_sid[0] = '\0'; + memset(&eng->session, 0, sizeof(eng->session)); eng->retry_count = 0; break; case EV_CMD_CONNECT: @@ -754,8 +755,8 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } // Once the primary peer is connected, transition to connected if (state == CONNECTION_STATE_CONNECTED) { - if ((role == PEER_ROLE_PUBLISHER && !eng->is_subscriber_primary) || - (role == PEER_ROLE_SUBSCRIBER && eng->is_subscriber_primary)) { + if ((role == PEER_ROLE_PUBLISHER && !eng->session.is_subscriber_primary) || + (role == PEER_ROLE_SUBSCRIBER && eng->session.is_subscriber_primary)) { eng->state = ENGINE_STATE_CONNECTED; } } From be7686220cdc1bbdd0d61ec779560735d6dc5a3f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:59:31 +1000 Subject: [PATCH 42/81] Reenable track subscription --- components/livekit/core/engine.c | 35 +++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 76bde72..4f0db8f 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -72,6 +72,7 @@ typedef struct { bool is_subscriber_primary; bool force_relay; livekit_pb_sid_t local_participant_sid; + livekit_pb_sid_t sub_audio_track_sid; } session_state_t; typedef struct { @@ -127,6 +128,27 @@ static inline void convert_dec_aud_info(esp_peer_audio_stream_info_t *info, av_r dec_info->bits_per_sample = 16; } +static engine_err_t subscribe_tracks(engine_t *eng, livekit_pb_track_info_t *tracks, int count) +{ + if (tracks == NULL || count <= 0) { + return ENGINE_ERR_INVALID_ARG; + } + if (eng->session.sub_audio_track_sid[0] != '\0') { + return ENGINE_ERR_NONE; + } + for (int i = 0; i < count; i++) { + livekit_pb_track_info_t *track = &tracks[i]; + if (track->type != LIVEKIT_PB_TRACK_TYPE_AUDIO) { + continue; + } + // For now, subscribe to the first audio track. + signal_send_update_subscription(eng->signal_handle, track->sid, true); + strncpy(eng->session.sub_audio_track_sid, track->sid, sizeof(eng->session.sub_audio_track_sid)); + break; + } + return ENGINE_ERR_NONE; +} + // MARK: - Published media /// Converts `esp_peer_audio_codec_t` to equivalent `esp_capture_codec_type_t` value. @@ -641,9 +663,6 @@ static void handle_room_update(engine_t *eng, livekit_pb_room_update_t *room_upd static void handle_participant_update(engine_t *eng, livekit_pb_participant_update_t *update) { - if (!eng->options.on_participant_info) { - return; - } bool found_local = false; for (pb_size_t i = 0; i < update->participants_count; i++) { livekit_pb_participant_info_t *participant = &update->participants[i]; @@ -652,8 +671,14 @@ static void handle_participant_update(engine_t *eng, livekit_pb_participant_upda eng->session.local_participant_sid, sizeof(eng->session.local_participant_sid) ) == 0; - if (is_local) found_local = true; - eng->options.on_participant_info(participant, is_local, eng->options.ctx); + if (is_local) { + found_local = true; + } else { + subscribe_tracks(eng, participant->tracks, participant->tracks_count); + } + if (eng->options.on_participant_info) { + eng->options.on_participant_info(participant, is_local, eng->options.ctx); + } } } From 67bb2e985297ec4b7a619da1549a42967a5af349 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 13:33:35 +1000 Subject: [PATCH 43/81] Decode tag on fail --- components/livekit/core/protocol.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index b3007d4..f400627 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -1,14 +1,28 @@ +#include #include "esp_log.h" #include "cJSON.h" #include "protocol.h" static const char *TAG = "livekit_protocol"; +static int32_t decode_first_tag(const pb_byte_t *buf, size_t len) +{ + pb_istream_t stream = pb_istream_from_buffer(buf, len); + pb_wire_type_t wire_type; + uint32_t tag; + bool eof = false; + if (!pb_decode_tag(&stream, &wire_type, &tag, &eof) || eof) { + return -1; + } + return (int32_t)tag; +} + bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, out)) { - ESP_LOGE(TAG, "Failed to decode data packet: %s", stream.errmsg); + ESP_LOGE(TAG, "Failed to decode data packet: type=%" PRId32 ", error=%s", + decode_first_tag(buf, len), stream.errmsg); return false; } return true; @@ -23,7 +37,8 @@ bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signa { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, out)) { - ESP_LOGE(TAG, "Failed to decode signal res: %s", stream.errmsg); + ESP_LOGE(TAG, "Failed to decode signal res: type=%" PRId32 ", error=%s", + decode_first_tag(buf, len), stream.errmsg); return false; } return true; From 9e1c5398f616f2f6b9b823eefc5c68ad894f62a0 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 13:42:05 +1000 Subject: [PATCH 44/81] Ignore proto messages --- components/livekit/protocol/livekit_rtc.pb.h | 28 ++++++++----------- .../protocol/protobufs/livekit_rtc.options | 6 ++-- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/components/livekit/protocol/livekit_rtc.pb.h b/components/livekit/protocol/livekit_rtc.pb.h index 19c3cba..d80d3bd 100644 --- a/components/livekit/protocol/livekit_rtc.pb.h +++ b/components/livekit/protocol/livekit_rtc.pb.h @@ -83,8 +83,7 @@ typedef struct livekit_pb_reconnect_response { } livekit_pb_reconnect_response_t; typedef struct livekit_pb_track_published_response { - char *cid; - livekit_pb_track_info_t track; + char dummy_field; } livekit_pb_track_published_response_t; typedef struct livekit_pb_track_unpublished_response { @@ -377,7 +376,7 @@ typedef struct livekit_pb_request_response { } livekit_pb_request_response_t; typedef struct livekit_pb_track_subscribed { - pb_callback_t track_sid; + char dummy_field; } livekit_pb_track_subscribed_t; typedef struct livekit_pb_signal_response { @@ -528,7 +527,7 @@ extern "C" { #define LIVEKIT_PB_MUTE_TRACK_REQUEST_INIT_DEFAULT {{{NULL}, NULL}, 0} #define LIVEKIT_PB_JOIN_RESPONSE_INIT_DEFAULT {false, LIVEKIT_PB_ROOM_INIT_DEFAULT, LIVEKIT_PB_PARTICIPANT_INFO_INIT_DEFAULT, 0, NULL, 0, {LIVEKIT_PB_ICE_SERVER_INIT_DEFAULT, LIVEKIT_PB_ICE_SERVER_INIT_DEFAULT, LIVEKIT_PB_ICE_SERVER_INIT_DEFAULT, LIVEKIT_PB_ICE_SERVER_INIT_DEFAULT}, 0, false, LIVEKIT_PB_CLIENT_CONFIGURATION_INIT_DEFAULT, 0, 0} #define LIVEKIT_PB_RECONNECT_RESPONSE_INIT_DEFAULT {{{NULL}, NULL}, false, LIVEKIT_PB_CLIENT_CONFIGURATION_INIT_DEFAULT, false, LIVEKIT_PB_SERVER_INFO_INIT_DEFAULT, 0} -#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_INIT_DEFAULT {NULL, LIVEKIT_PB_TRACK_INFO_INIT_DEFAULT} +#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_INIT_DEFAULT {0} #define LIVEKIT_PB_TRACK_UNPUBLISHED_RESPONSE_INIT_DEFAULT {{{NULL}, NULL}} #define LIVEKIT_PB_SESSION_DESCRIPTION_INIT_DEFAULT {"", NULL, 0} #define LIVEKIT_PB_PARTICIPANT_UPDATE_INIT_DEFAULT {0, NULL} @@ -563,7 +562,7 @@ extern "C" { #define LIVEKIT_PB_REGION_INFO_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}, 0} #define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_INIT_DEFAULT {{{NULL}, NULL}, _LIVEKIT_PB_SUBSCRIPTION_ERROR_MIN} #define LIVEKIT_PB_REQUEST_RESPONSE_INIT_DEFAULT {0, _LIVEKIT_PB_REQUEST_RESPONSE_REASON_MIN, {{NULL}, NULL}} -#define LIVEKIT_PB_TRACK_SUBSCRIBED_INIT_DEFAULT {{{NULL}, NULL}} +#define LIVEKIT_PB_TRACK_SUBSCRIBED_INIT_DEFAULT {0} #define LIVEKIT_PB_SIGNAL_REQUEST_INIT_ZERO {0, {LIVEKIT_PB_SESSION_DESCRIPTION_INIT_ZERO}} #define LIVEKIT_PB_SIGNAL_RESPONSE_INIT_ZERO {0, {LIVEKIT_PB_JOIN_RESPONSE_INIT_ZERO}} #define LIVEKIT_PB_SIMULCAST_CODEC_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}} @@ -572,7 +571,7 @@ extern "C" { #define LIVEKIT_PB_MUTE_TRACK_REQUEST_INIT_ZERO {{{NULL}, NULL}, 0} #define LIVEKIT_PB_JOIN_RESPONSE_INIT_ZERO {false, LIVEKIT_PB_ROOM_INIT_ZERO, LIVEKIT_PB_PARTICIPANT_INFO_INIT_ZERO, 0, NULL, 0, {LIVEKIT_PB_ICE_SERVER_INIT_ZERO, LIVEKIT_PB_ICE_SERVER_INIT_ZERO, LIVEKIT_PB_ICE_SERVER_INIT_ZERO, LIVEKIT_PB_ICE_SERVER_INIT_ZERO}, 0, false, LIVEKIT_PB_CLIENT_CONFIGURATION_INIT_ZERO, 0, 0} #define LIVEKIT_PB_RECONNECT_RESPONSE_INIT_ZERO {{{NULL}, NULL}, false, LIVEKIT_PB_CLIENT_CONFIGURATION_INIT_ZERO, false, LIVEKIT_PB_SERVER_INFO_INIT_ZERO, 0} -#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_INIT_ZERO {NULL, LIVEKIT_PB_TRACK_INFO_INIT_ZERO} +#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_INIT_ZERO {0} #define LIVEKIT_PB_TRACK_UNPUBLISHED_RESPONSE_INIT_ZERO {{{NULL}, NULL}} #define LIVEKIT_PB_SESSION_DESCRIPTION_INIT_ZERO {"", NULL, 0} #define LIVEKIT_PB_PARTICIPANT_UPDATE_INIT_ZERO {0, NULL} @@ -607,7 +606,7 @@ extern "C" { #define LIVEKIT_PB_REGION_INFO_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}, 0} #define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_INIT_ZERO {{{NULL}, NULL}, _LIVEKIT_PB_SUBSCRIPTION_ERROR_MIN} #define LIVEKIT_PB_REQUEST_RESPONSE_INIT_ZERO {0, _LIVEKIT_PB_REQUEST_RESPONSE_REASON_MIN, {{NULL}, NULL}} -#define LIVEKIT_PB_TRACK_SUBSCRIBED_INIT_ZERO {{{NULL}, NULL}} +#define LIVEKIT_PB_TRACK_SUBSCRIBED_INIT_ZERO {0} /* Field tags (for use in manual encoding/decoding) */ #define LIVEKIT_PB_SIMULCAST_CODEC_CODEC_TAG 1 @@ -628,8 +627,6 @@ extern "C" { #define LIVEKIT_PB_RECONNECT_RESPONSE_CLIENT_CONFIGURATION_TAG 2 #define LIVEKIT_PB_RECONNECT_RESPONSE_SERVER_INFO_TAG 3 #define LIVEKIT_PB_RECONNECT_RESPONSE_LAST_MESSAGE_SEQ_TAG 4 -#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_CID_TAG 1 -#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_TRACK_TAG 2 #define LIVEKIT_PB_TRACK_UNPUBLISHED_RESPONSE_TRACK_SID_TAG 1 #define LIVEKIT_PB_SESSION_DESCRIPTION_TYPE_TAG 1 #define LIVEKIT_PB_SESSION_DESCRIPTION_SDP_TAG 2 @@ -747,7 +744,6 @@ extern "C" { #define LIVEKIT_PB_REQUEST_RESPONSE_REQUEST_ID_TAG 1 #define LIVEKIT_PB_REQUEST_RESPONSE_REASON_TAG 2 #define LIVEKIT_PB_REQUEST_RESPONSE_MESSAGE_TAG 3 -#define LIVEKIT_PB_TRACK_SUBSCRIBED_TRACK_SID_TAG 1 #define LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG 1 #define LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG 2 #define LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG 3 @@ -916,11 +912,9 @@ X(a, STATIC, SINGULAR, UINT32, last_message_seq, 4) #define livekit_pb_reconnect_response_t_server_info_MSGTYPE livekit_pb_server_info_t #define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_FIELDLIST(X, a) \ -X(a, POINTER, SINGULAR, STRING, cid, 1) \ -X(a, STATIC, REQUIRED, MESSAGE, track, 2) + #define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_CALLBACK NULL #define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_DEFAULT NULL -#define livekit_pb_track_published_response_t_track_MSGTYPE livekit_pb_track_info_t #define LIVEKIT_PB_TRACK_UNPUBLISHED_RESPONSE_FIELDLIST(X, a) \ X(a, CALLBACK, SINGULAR, STRING, track_sid, 1) @@ -1171,8 +1165,8 @@ X(a, CALLBACK, SINGULAR, STRING, message, 3) #define LIVEKIT_PB_REQUEST_RESPONSE_DEFAULT NULL #define LIVEKIT_PB_TRACK_SUBSCRIBED_FIELDLIST(X, a) \ -X(a, CALLBACK, SINGULAR, STRING, track_sid, 1) -#define LIVEKIT_PB_TRACK_SUBSCRIBED_CALLBACK pb_default_field_callback + +#define LIVEKIT_PB_TRACK_SUBSCRIBED_CALLBACK NULL #define LIVEKIT_PB_TRACK_SUBSCRIBED_DEFAULT NULL extern const pb_msgdesc_t livekit_pb_signal_request_t_msg; @@ -1274,7 +1268,6 @@ extern const pb_msgdesc_t livekit_pb_track_subscribed_t_msg; /* livekit_pb_MuteTrackRequest_size depends on runtime parameters */ /* livekit_pb_JoinResponse_size depends on runtime parameters */ /* livekit_pb_ReconnectResponse_size depends on runtime parameters */ -/* livekit_pb_TrackPublishedResponse_size depends on runtime parameters */ /* livekit_pb_TrackUnpublishedResponse_size depends on runtime parameters */ /* livekit_pb_SessionDescription_size depends on runtime parameters */ /* livekit_pb_ParticipantUpdate_size depends on runtime parameters */ @@ -1303,7 +1296,6 @@ extern const pb_msgdesc_t livekit_pb_track_subscribed_t_msg; /* livekit_pb_RegionInfo_size depends on runtime parameters */ /* livekit_pb_SubscriptionResponse_size depends on runtime parameters */ /* livekit_pb_RequestResponse_size depends on runtime parameters */ -/* livekit_pb_TrackSubscribed_size depends on runtime parameters */ #define LIVEKIT_LIVEKIT_RTC_PB_H_MAX_SIZE LIVEKIT_PB_ADD_TRACK_REQUEST_SIZE #define LIVEKIT_PB_ADD_TRACK_REQUEST_SIZE 81 #define LIVEKIT_PB_LEAVE_REQUEST_SIZE 4 @@ -1311,6 +1303,8 @@ extern const pb_msgdesc_t livekit_pb_track_subscribed_t_msg; #define LIVEKIT_PB_PONG_SIZE 22 #define LIVEKIT_PB_SIMULATE_SCENARIO_SIZE 11 #define LIVEKIT_PB_SUBSCRIBED_QUALITY_SIZE 4 +#define LIVEKIT_PB_TRACK_PUBLISHED_RESPONSE_SIZE 0 +#define LIVEKIT_PB_TRACK_SUBSCRIBED_SIZE 0 #if defined(livekit_pb_Room_size) #define LIVEKIT_PB_ROOM_UPDATE_SIZE (6 + livekit_pb_Room_size) #endif diff --git a/components/livekit/protocol/protobufs/livekit_rtc.options b/components/livekit/protocol/protobufs/livekit_rtc.options index bbf48cd..749a32c 100644 --- a/components/livekit/protocol/protobufs/livekit_rtc.options +++ b/components/livekit/protocol/protobufs/livekit_rtc.options @@ -35,8 +35,10 @@ livekit_pb.AddTrackRequest.stream type:FT_IGNORE livekit_pb.AddTrackRequest.backup_codec_policy type:FT_IGNORE livekit_pb.AddTrackRequest.audio_features max_count:8 -livekit_pb.TrackPublishedResponse.cid type:FT_POINTER -livekit_pb.TrackPublishedResponse.track label_override:LABEL_REQUIRED +livekit_pb.TrackPublishedResponse.cid type:FT_IGNORE +livekit_pb.TrackPublishedResponse.track type:FT_IGNORE + +livekit_pb.TrackSubscribed.track_sid type:FT_IGNORE livekit_pb.ParticipantUpdate.participants type:FT_POINTER From da0429466795bb514d0b16a9305d80e75f1759c1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:08:24 +1000 Subject: [PATCH 45/81] Add markers --- components/livekit/core/engine.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 4f0db8f..d4ed695 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -682,7 +682,7 @@ static void handle_participant_update(engine_t *eng, livekit_pb_participant_upda } } -// MARK: - FSM state handlers +// MARK: - State: Disconnected /// Handler for `ENGINE_STATE_DISCONNECTED`. static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) @@ -711,6 +711,8 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) return false; } +// MARK: - State: Connecting + /// Handler for `ENGINE_STATE_CONNECTING`. static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) { @@ -792,6 +794,8 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) return false; } +// MARK: - State: Connected + /// Handler for `ENGINE_STATE_CONNECTED`. static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) { @@ -869,6 +873,8 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) return false; } +// MARK: - State: Backoff + /// Handler for `ENGINE_STATE_BACKOFF`. static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) { From fc14eb05d23f48dd83001d3cfeebece42e212ec1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:02:42 +1000 Subject: [PATCH 46/81] Fix state reporting bug Optimizer causes issue where enum return value cannot be -1 --- components/livekit/core/engine.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index d4ed695..de0020a 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -496,28 +496,37 @@ static bool establish_peer_connections(engine_t *eng) // MARK: - FSM helpers -/// Return the external state that should be reported based on the engine's current state. +/// Determines the external state that should be reported. /// /// This is necessary because the engine FSM's states do not map 1:1 with the states /// exposed in the public room API. /// -static inline livekit_connection_state_t map_engine_state(engine_t *eng) +static inline bool map_engine_state(engine_t *eng, livekit_connection_state_t *out_state) { switch (eng->state) { case ENGINE_STATE_DISCONNECTED: // Engine state machine doesn't have a discrete failed state - return eng->failure_reason == LIVEKIT_FAILURE_REASON_NONE ? + *out_state = eng->failure_reason == LIVEKIT_FAILURE_REASON_NONE ? LIVEKIT_CONNECTION_STATE_DISCONNECTED : LIVEKIT_CONNECTION_STATE_FAILED; + break; case ENGINE_STATE_CONNECTING: // Should only report connecting for initial connection attempt. - return eng->retry_count <= 0 ? LIVEKIT_CONNECTION_STATE_CONNECTING : -1; + if (eng->retry_count > 0) { + return false; + } + *out_state = LIVEKIT_CONNECTION_STATE_CONNECTING; + break; case ENGINE_STATE_BACKOFF: - return LIVEKIT_CONNECTION_STATE_RECONNECTING; + *out_state = LIVEKIT_CONNECTION_STATE_RECONNECTING; + break; case ENGINE_STATE_CONNECTED: - return LIVEKIT_CONNECTION_STATE_CONNECTED; - default: return -1; + *out_state = LIVEKIT_CONNECTION_STATE_CONNECTED; + break; + default: + return false; } + return true; } /// Map a signal failure reason to a failure reason exposed in the public room API. @@ -961,8 +970,10 @@ static void engine_task(void *arg) assert(eng->state == state); if (eng->options.on_state_changed) { - livekit_connection_state_t ext_state = map_engine_state(eng); - if (ext_state > 0) eng->options.on_state_changed(ext_state, eng->options.ctx); + livekit_connection_state_t ext_state; + if (map_engine_state(eng, &ext_state)) { + eng->options.on_state_changed(ext_state, eng->options.ctx); + } } } } From e5322d32a8e16ba99f40b1d02c59ac46a9880505 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:28:08 +1000 Subject: [PATCH 47/81] Improve signal state handling - Signal state enum with failure reason - Handle normal WS closure --- components/livekit/core/common.h | 12 +++--- components/livekit/core/engine.c | 50 ++++++++++++++---------- components/livekit/core/signaling.c | 59 ++++++++++++++--------------- components/livekit/core/signaling.h | 54 +++++++++++++++----------- 4 files changed, 94 insertions(+), 81 deletions(-) diff --git a/components/livekit/core/common.h b/components/livekit/core/common.h index 2d65356..5465234 100644 --- a/components/livekit/core/common.h +++ b/components/livekit/core/common.h @@ -9,13 +9,13 @@ extern "C" { #endif -/// State of an engine component or the engine itself. +/// State of a connection. typedef enum { - CONNECTION_STATE_DISCONNECTED = 0, /*!< Disconnected */ - CONNECTION_STATE_CONNECTING = 1, /*!< Establishing connection */ - CONNECTION_STATE_CONNECTED = 2, /*!< Connected */ - CONNECTION_STATE_RECONNECTING = 3, /*!< Connection was previously established, but was lost */ - CONNECTION_STATE_FAILED = 4 /*!< Connection failed */ + CONNECTION_STATE_DISCONNECTED = 0, /// Disconnected + CONNECTION_STATE_CONNECTING = 1, /// Establishing connection + CONNECTION_STATE_CONNECTED = 2, /// Connected + CONNECTION_STATE_RECONNECTING = 3, /// Connection was previously established, but was lost + CONNECTION_STATE_FAILED = 4 /// Connection failed } connection_state_t; typedef struct { diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index de0020a..68d0645 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -58,7 +58,7 @@ typedef struct { livekit_pb_data_packet_t data_packet; /// Detail for `EV_SIG_STATE`. - connection_state_t sig_state; + signal_state_t sig_state; /// Detail for `EV_PEER_STATE`. struct { @@ -329,7 +329,7 @@ static engine_err_t publish_tracks(engine_t *eng) // MARK: - Signal event handlers -static void on_signal_state_changed(connection_state_t state, void *ctx) +static void on_signal_state_changed(signal_state_t state, void *ctx) { engine_t *eng = (engine_t *)ctx; engine_event_t ev = { @@ -529,14 +529,14 @@ static inline bool map_engine_state(engine_t *eng, livekit_connection_state_t *o return true; } -/// Map a signal failure reason to a failure reason exposed in the public room API. -static livekit_failure_reason_t map_signal_failure_reason(signal_failure_reason_t reason) +/// Map a signal failed state to a failure reason exposed in the public room API. +static livekit_failure_reason_t map_signal_fail_state(signal_state_t state) { - switch (reason) { - case SIGNAL_FAILURE_REASON_UNREACHABLE: return LIVEKIT_FAILURE_REASON_UNREACHABLE; - case SIGNAL_FAILURE_REASON_BAD_TOKEN: return LIVEKIT_FAILURE_REASON_BAD_TOKEN; - case SIGNAL_FAILURE_REASON_UNAUTHORIZED: return LIVEKIT_FAILURE_REASON_UNAUTHORIZED; - default: return LIVEKIT_FAILURE_REASON_OTHER; + switch (state) { + case SIGNAL_STATE_FAILED_UNREACHABLE: return LIVEKIT_FAILURE_REASON_UNREACHABLE; + case SIGNAL_STATE_FAILED_BAD_TOKEN: return LIVEKIT_FAILURE_REASON_BAD_TOKEN; + case SIGNAL_STATE_FAILED_UNAUTHORIZED: return LIVEKIT_FAILURE_REASON_UNAUTHORIZED; + default: return LIVEKIT_FAILURE_REASON_OTHER; } } @@ -770,27 +770,29 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - if (ev->detail.sig_state == CONNECTION_STATE_FAILED) { - signal_failure_reason_t reason = signal_get_failure_reason(eng->signal_handle); - eng->failure_reason = map_signal_failure_reason(reason); - // Client errors should not trigger a reconnection - eng->state = (reason & SIGNAL_FAILURE_REASON_CLIENT_ANY) ? + signal_state_t sig_state = ev->detail.sig_state; + if (sig_state == SIGNAL_STATE_DISCONNECTED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_OTHER; + eng->state = ENGINE_STATE_BACKOFF; + } else if (sig_state & SIGNAL_STATE_FAILED_ANY) { + eng->failure_reason = map_signal_fail_state(sig_state); + eng->state = (sig_state & SIGNAL_STATE_FAILED_CLIENT_ANY) ? ENGINE_STATE_DISCONNECTED : ENGINE_STATE_BACKOFF; } break; case EV_PEER_STATE: - connection_state_t state = ev->detail.peer_state.state; + connection_state_t peer_state = ev->detail.peer_state.state; peer_role_t role = ev->detail.peer_state.role; // If either peer fails, transition to backoff - if (state == CONNECTION_STATE_FAILED) { + if (peer_state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; break; } // Once the primary peer is connected, transition to connected - if (state == CONNECTION_STATE_CONNECTED) { + if (peer_state == CONNECTION_STATE_CONNECTED) { if ((role == PEER_ROLE_PUBLISHER && !eng->session.is_subscriber_primary) || (role == PEER_ROLE_SUBSCRIBER && eng->session.is_subscriber_primary)) { eng->state = ENGINE_STATE_CONNECTED; @@ -861,17 +863,23 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) } break; case EV_SIG_STATE: - if (ev->detail.sig_state == CONNECTION_STATE_FAILED) { - eng->failure_reason = LIVEKIT_FAILURE_REASON_UNREACHABLE; + signal_state_t sig_state = ev->detail.sig_state; + if (sig_state == SIGNAL_STATE_DISCONNECTED) { + eng->failure_reason = LIVEKIT_FAILURE_REASON_OTHER; eng->state = ENGINE_STATE_BACKOFF; + } else if (sig_state & SIGNAL_STATE_FAILED_ANY) { + eng->failure_reason = map_signal_fail_state(sig_state); + eng->state = (sig_state & SIGNAL_STATE_FAILED_CLIENT_ANY) ? + ENGINE_STATE_DISCONNECTED : + ENGINE_STATE_BACKOFF; } break; case EV_PEER_STATE: - connection_state_t state = ev->detail.peer_state.state; + connection_state_t peer_state = ev->detail.peer_state.state; peer_role_t role = ev->detail.peer_state.role; // If either peer fails, transition to backoff - if (state == CONNECTION_STATE_FAILED) { + if (peer_state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 07cfd0b..3d6ac98 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -29,21 +29,26 @@ typedef struct { esp_websocket_client_handle_t ws; signal_options_t options; esp_timer_handle_t ping_timer; - signal_failure_reason_t failure_reason; + bool last_attempt_failed; int32_t ping_interval_ms; int32_t ping_timeout_ms; int64_t rtt; } signal_t; -static inline signal_failure_reason_t failure_reason_from_http_status(int status) +static inline void state_changed(signal_t *sg, signal_state_t state) +{ + sg->options.on_state_changed(state, sg->options.ctx); +} + +static inline signal_state_t failed_state_from_http_status(int status) { switch (status) { - case 400: return SIGNAL_FAILURE_REASON_BAD_TOKEN; - case 401: return SIGNAL_FAILURE_REASON_UNAUTHORIZED; + case 400: return SIGNAL_STATE_FAILED_BAD_TOKEN; + case 401: return SIGNAL_STATE_FAILED_UNAUTHORIZED; default: return status > 400 && status < 500 ? - SIGNAL_FAILURE_REASON_CLIENT_OTHER : - SIGNAL_FAILURE_REASON_INTERNAL; + SIGNAL_STATE_FAILED_CLIENT_OTHER : + SIGNAL_STATE_FAILED_INTERNAL; } } @@ -129,16 +134,25 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void switch (event_id) { case WEBSOCKET_EVENT_BEFORE_CONNECT: - sg->failure_reason = SIGNAL_FAILURE_REASON_NONE; - break; - case WEBSOCKET_EVENT_CONNECTED: - ESP_LOGD(TAG, "Signaling connected"); - sg->options.on_state_changed(CONNECTION_STATE_CONNECTED, sg->options.ctx); + sg->last_attempt_failed = false; + state_changed(sg, SIGNAL_STATE_CONNECTING); break; + case WEBSOCKET_EVENT_CLOSED: case WEBSOCKET_EVENT_DISCONNECTED: - ESP_LOGD(TAG, "Signaling disconnected"); esp_timer_stop(sg->ping_timer); - sg->options.on_state_changed(CONNECTION_STATE_DISCONNECTED, sg->options.ctx); + if (!sg->last_attempt_failed) { + state_changed(sg, SIGNAL_STATE_DISCONNECTED); + } + break; + case WEBSOCKET_EVENT_ERROR: + int http_status = data->error_handle.esp_ws_handshake_status_code; + signal_state_t state = http_status != 0 ? + failed_state_from_http_status(http_status) : + SIGNAL_STATE_FAILED_UNREACHABLE; + state_changed(sg, state); + break; + case WEBSOCKET_EVENT_CONNECTED: + state_changed(sg, SIGNAL_STATE_CONNECTED); break; case WEBSOCKET_EVENT_DATA: if (data->op_code != WS_TRANSPORT_OPCODES_BINARY) { @@ -159,16 +173,8 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void protocol_signal_res_free(&res); } break; - case WEBSOCKET_EVENT_ERROR: - esp_timer_stop(sg->ping_timer); - - int http_status = data->error_handle.esp_ws_handshake_status_code; - sg->failure_reason = http_status != 0 ? - failure_reason_from_http_status(http_status) : - SIGNAL_FAILURE_REASON_UNREACHABLE; - sg->options.on_state_changed(CONNECTION_STATE_FAILED, sg->options.ctx); + default: break; - default: break; } } @@ -283,15 +289,6 @@ signal_err_t signal_close(signal_handle_t handle) return SIGNAL_ERR_NONE; } -signal_failure_reason_t signal_get_failure_reason(signal_handle_t handle) -{ - if (handle == NULL) { - return SIGNAL_FAILURE_REASON_NONE; - } - signal_t *sg = (signal_t *)handle; - return sg->failure_reason; -} - signal_err_t signal_send_leave(signal_handle_t handle) { if (handle == NULL) { diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index 710962d..a69f79c 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -21,36 +21,51 @@ typedef enum { // TODO: Add more error cases as needed } signal_err_t; -/// Reason why signal connection failed. +/// Signal connection state. typedef enum { - /// No failure has occurred. - SIGNAL_FAILURE_REASON_NONE = 0, + /// Unknown state. + SIGNAL_STATE_UNKNOWN = 0, + + /// Disconnected. + SIGNAL_STATE_DISCONNECTED = 1 << 0, + + /// Establishing connection. + SIGNAL_STATE_CONNECTING = 1 << 1, + + /// Connection established. + SIGNAL_STATE_CONNECTED = 1 << 2, /// Server unreachable. - SIGNAL_FAILURE_REASON_UNREACHABLE = 1 << 0, + SIGNAL_STATE_FAILED_UNREACHABLE = 1 << 3, + + /// Internal server error. + SIGNAL_STATE_FAILED_INTERNAL = 1 << 4, /// Token is malformed. - SIGNAL_FAILURE_REASON_BAD_TOKEN = 1 << 1, + SIGNAL_STATE_FAILED_BAD_TOKEN = 1 << 5, /// Token is not valid to join the room. - SIGNAL_FAILURE_REASON_UNAUTHORIZED = 1 << 2, + SIGNAL_STATE_FAILED_UNAUTHORIZED = 1 << 6, - /// Other client error not covered by other reasons. - SIGNAL_FAILURE_REASON_CLIENT_OTHER = 1 << 3, + /// Other client failure not covered by other reasons. + SIGNAL_STATE_FAILED_CLIENT_OTHER = 1 << 7, - /// Any client error, no retry should be attempted. - SIGNAL_FAILURE_REASON_CLIENT_ANY = SIGNAL_FAILURE_REASON_BAD_TOKEN | - SIGNAL_FAILURE_REASON_UNAUTHORIZED | - SIGNAL_FAILURE_REASON_CLIENT_OTHER, - /// Internal server error. - SIGNAL_FAILURE_REASON_INTERNAL = 1 << 4 -} signal_failure_reason_t; + /// Any client failure (retry should not be attempted). + SIGNAL_STATE_FAILED_CLIENT_ANY = SIGNAL_STATE_FAILED_BAD_TOKEN | + SIGNAL_STATE_FAILED_UNAUTHORIZED | + SIGNAL_STATE_FAILED_CLIENT_OTHER, + + /// Any failure. + SIGNAL_STATE_FAILED_ANY = SIGNAL_STATE_FAILED_UNREACHABLE | + SIGNAL_STATE_FAILED_INTERNAL | + SIGNAL_STATE_FAILED_CLIENT_ANY +} signal_state_t; typedef struct { void* ctx; /// Invoked when the connection state changes. - void (*on_state_changed)(connection_state_t state, void *ctx); + void (*on_state_changed)(signal_state_t state, void *ctx); /// Invoked when a signal response is received. /// @@ -71,13 +86,6 @@ signal_err_t signal_connect(signal_handle_t handle, const char* server_url, cons /// Closes the WebSocket connection signal_err_t signal_close(signal_handle_t handle); -/// Returns the reason why the connection failed. -/// -/// Use after the signal client's state changes to `CONNECTION_STATE_FAILED`. -/// Will be reset to `SIGNAL_FAILURE_REASON_NONE` during the next connection attempt. -/// -signal_failure_reason_t signal_get_failure_reason(signal_handle_t handle); - /// Sends a leave request. signal_err_t signal_send_leave(signal_handle_t handle); signal_err_t signal_send_offer(signal_handle_t handle, const char *sdp); From 37385e250f31c2018984e519f28a80427d175c17 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:30:07 +1000 Subject: [PATCH 48/81] Remove unneeded includes --- components/livekit/core/signaling.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 3d6ac98..a6ed92d 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -1,10 +1,5 @@ -#include -#include -#define LOG_LOCAL_LEVEL ESP_LOG_DEBUG #include "esp_log.h" -#include "esp_peer_signaling.h" #include "esp_netif.h" -#include "media_lib_os.h" #ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE #include "esp_crt_bundle.h" #endif From 488bb0f2a9eebe2940ae8e36cbd06f96bde7ad7f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:42:31 +1000 Subject: [PATCH 49/81] Retry on peer disconnect in addition to fail --- components/livekit/core/engine.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 68d0645..716aeee 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -785,8 +785,9 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) connection_state_t peer_state = ev->detail.peer_state.state; peer_role_t role = ev->detail.peer_state.role; - // If either peer fails, transition to backoff - if (peer_state == CONNECTION_STATE_FAILED) { + // If either peer fails or disconnects, transition to backoff + if (peer_state == CONNECTION_STATE_DISCONNECTED || + peer_state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; break; @@ -878,8 +879,9 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) connection_state_t peer_state = ev->detail.peer_state.state; peer_role_t role = ev->detail.peer_state.role; - // If either peer fails, transition to backoff - if (peer_state == CONNECTION_STATE_FAILED) { + // If either peer fail or disconnects, transition to backoff + if (peer_state == CONNECTION_STATE_DISCONNECTED || + peer_state == CONNECTION_STATE_FAILED) { eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } From 72f459eb41eb756b20d25417627d86dc1afaace3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:45:36 +1000 Subject: [PATCH 50/81] Remove unneeded config --- components/livekit/core/signaling.c | 1 - 1 file changed, 1 deletion(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index a6ed92d..06e89ad 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -206,7 +206,6 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) static esp_websocket_client_config_t ws_config = { .buffer_size = SIGNAL_WS_BUFFER_SIZE, .disable_pingpong_discon = true, - .reconnect_timeout_ms = SIGNAL_WS_RECONNECT_TIMEOUT_MS, .network_timeout_ms = SIGNAL_WS_NETWORK_TIMEOUT_MS, .disable_auto_reconnect = true, #ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE From 5ea28d536c5d027fa99084f6e31b63bf3b57342b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:46:27 +1000 Subject: [PATCH 51/81] Refactor signal client - Handle ping timeout - Use FreeRTOS timer instead of esp_timer (ms accuracy is sufficient) - Ensure cleanup under abnormal closure conditions (e.g. WiFi disconnect) --- components/livekit/CMakeLists.txt | 1 - components/livekit/core/engine.c | 1 + components/livekit/core/livekit.c | 1 + components/livekit/core/signaling.c | 117 +++++++++++++-------- components/livekit/core/signaling.h | 22 ++-- components/livekit/include/livekit_types.h | 6 ++ 6 files changed, 92 insertions(+), 56 deletions(-) diff --git a/components/livekit/CMakeLists.txt b/components/livekit/CMakeLists.txt index 66cf763..cdf6c5f 100644 --- a/components/livekit/CMakeLists.txt +++ b/components/livekit/CMakeLists.txt @@ -8,7 +8,6 @@ idf_component_register( PRIV_REQUIRES esp_codec_dev esp_netif - esp_timer esp_websocket_client esp_webrtc json diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 716aeee..4c6a970 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -534,6 +534,7 @@ static livekit_failure_reason_t map_signal_fail_state(signal_state_t state) { switch (state) { case SIGNAL_STATE_FAILED_UNREACHABLE: return LIVEKIT_FAILURE_REASON_UNREACHABLE; + case SIGNAL_STATE_FAILED_PING_TIMEOUT: return LIVEKIT_FAILURE_REASON_PING_TIMEOUT; case SIGNAL_STATE_FAILED_BAD_TOKEN: return LIVEKIT_FAILURE_REASON_BAD_TOKEN; case SIGNAL_STATE_FAILED_UNAUTHORIZED: return LIVEKIT_FAILURE_REASON_UNAUTHORIZED; default: return LIVEKIT_FAILURE_REASON_OTHER; diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index b47a14f..c190ae3 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -304,6 +304,7 @@ const char* livekit_failure_reason_str(livekit_failure_reason_t reason) case LIVEKIT_FAILURE_REASON_UNAUTHORIZED: return "Unauthorized"; case LIVEKIT_FAILURE_REASON_RTC: return "RTC"; case LIVEKIT_FAILURE_REASON_MAX_RETRIES: return "Max Retries"; + case LIVEKIT_FAILURE_REASON_PING_TIMEOUT: return "Ping Timeout"; case LIVEKIT_FAILURE_REASON_DUPLICATE_IDENTITY: return "Duplicate Identity"; case LIVEKIT_FAILURE_REASON_SERVER_SHUTDOWN: return "Server Shutdown"; case LIVEKIT_FAILURE_REASON_PARTICIPANT_REMOVED: return "Participant Removed"; diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 06e89ad..3ef2d90 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -1,3 +1,5 @@ +#include "freertos/FreeRTOS.h" +#include "freertos/timers.h" #include "esp_log.h" #include "esp_netif.h" #ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE @@ -5,7 +7,6 @@ #endif #include "esp_websocket_client.h" #include "esp_tls.h" -#include "esp_timer.h" #include "protocol.h" #include "signaling.h" @@ -22,17 +23,17 @@ static const char *TAG = "livekit_signaling"; typedef struct { esp_websocket_client_handle_t ws; - signal_options_t options; - esp_timer_handle_t ping_timer; - bool last_attempt_failed; - - int32_t ping_interval_ms; - int32_t ping_timeout_ms; + signal_options_t options; + signal_state_t state; + bool is_terminal_state; + TimerHandle_t ping_interval_timer; + TimerHandle_t ping_timeout_timer; int64_t rtt; } signal_t; -static inline void state_changed(signal_t *sg, signal_state_t state) +static inline void change_state(signal_t *sg, signal_state_t state) { + sg->state = state; sg->options.on_state_changed(state, sg->options.ctx); } @@ -81,9 +82,9 @@ static signal_err_t send_request(signal_t *sg, livekit_pb_signal_request_t *requ return ret; } -static void send_ping(void *arg) +static void on_ping_interval_expired(TimerHandle_t handle) { - signal_t *sg = (signal_t *)arg; + signal_t *sg = (signal_t *)pvTimerGetTimerID(handle); livekit_pb_signal_request_t req = LIVEKIT_PB_SIGNAL_REQUEST_INIT_DEFAULT; req.which_message = LIVEKIT_PB_SIGNAL_REQUEST_PING_REQ_TAG; @@ -93,6 +94,12 @@ static void send_ping(void *arg) send_request(sg, &req); } +static void on_ping_timeout_expired(TimerHandle_t handle) +{ + signal_t *sg = (signal_t *)pvTimerGetTimerID(handle); + esp_websocket_client_stop(sg->ws); +} + /// Processes responses before forwarding them to the receiver. static inline bool res_middleware(signal_t *sg, livekit_pb_signal_response_t *res) { @@ -100,54 +107,65 @@ static inline bool res_middleware(signal_t *sg, livekit_pb_signal_response_t *re res->which_message != LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG) { return true; } - bool should_forward = false; switch (res->which_message) { + case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: + livekit_pb_join_response_t *join = &res->message.join; + // Calculate timer intervals and start timers: seconds -> ms, min 1s. + int32_t ping_interval_ms = (join->ping_interval < 1 ? 1 : join->ping_interval) * 1000; + xTimerChangePeriod(sg->ping_interval_timer, pdMS_TO_TICKS(ping_interval_ms), 0); + xTimerStart(sg->ping_interval_timer, 0); + int32_t ping_timeout_ms = (join->ping_timeout < 1 ? 1 : join->ping_timeout) * 1000; + xTimerChangePeriod(sg->ping_timeout_timer, pdMS_TO_TICKS(ping_timeout_ms), 0); + xTimerStart(sg->ping_timeout_timer, 0); + return true; case LIVEKIT_PB_SIGNAL_RESPONSE_PONG_RESP_TAG: livekit_pb_pong_t *pong = &res->message.pong_resp; + // Calculate round trip time (RTT) and restart ping timeout timer. sg->rtt = get_unix_time_ms() - pong->last_ping_timestamp; - // TODO: Reset ping timeout - should_forward = false; - break; - case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: - livekit_pb_join_response_t *join = &res->message.join; - sg->ping_interval_ms = join->ping_interval * 1000; - sg->ping_timeout_ms = join->ping_timeout * 1000; - esp_timer_start_periodic(sg->ping_timer, sg->ping_interval_ms * 1000); - should_forward = true; - break; + xTimerReset(sg->ping_timeout_timer, 0); + return false; default: - should_forward = false; + return true; } - return should_forward; } static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void *event_data) { - assert(ctx != NULL); signal_t *sg = (signal_t *)ctx; esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; switch (event_id) { case WEBSOCKET_EVENT_BEFORE_CONNECT: - sg->last_attempt_failed = false; - state_changed(sg, SIGNAL_STATE_CONNECTING); + sg->is_terminal_state = false; + change_state(sg, SIGNAL_STATE_CONNECTING); break; case WEBSOCKET_EVENT_CLOSED: case WEBSOCKET_EVENT_DISCONNECTED: - esp_timer_stop(sg->ping_timer); - if (!sg->last_attempt_failed) { - state_changed(sg, SIGNAL_STATE_DISCONNECTED); + case WEBSOCKET_EVENT_FINISH: + if (sg->is_terminal_state) { + break; + } + bool is_ping_timeout = xTimerIsTimerActive(sg->ping_timeout_timer) == pdFALSE; + xTimerStop(sg->ping_timeout_timer, 0); + xTimerStop(sg->ping_interval_timer, 0); + + if (!(sg->state & SIGNAL_STATE_FAILED_ANY)) { + signal_state_t terminal_state = is_ping_timeout ? + SIGNAL_STATE_FAILED_PING_TIMEOUT : + SIGNAL_STATE_DISCONNECTED; + change_state(sg, terminal_state); } + sg->is_terminal_state = true; break; case WEBSOCKET_EVENT_ERROR: int http_status = data->error_handle.esp_ws_handshake_status_code; signal_state_t state = http_status != 0 ? failed_state_from_http_status(http_status) : SIGNAL_STATE_FAILED_UNREACHABLE; - state_changed(sg, state); + change_state(sg, state); break; case WEBSOCKET_EVENT_CONNECTED: - state_changed(sg, SIGNAL_STATE_CONNECTED); + change_state(sg, SIGNAL_STATE_CONNECTED); break; case WEBSOCKET_EVENT_DATA: if (data->op_code != WS_TRANSPORT_OPCODES_BINARY) { @@ -191,13 +209,26 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) } sg->options = *options; - esp_timer_create_args_t timer_args = { - .callback = send_ping, - .arg = sg, - .name = "ping" - }; - if (esp_timer_create(&timer_args, &sg->ping_timer) != ESP_OK) { - ESP_LOGE(TAG, "Failed to create ping timer"); + sg->ping_interval_timer = xTimerCreate( + "ping_interval", + pdMS_TO_TICKS(1000), // Will be overwritten before start + pdTRUE, // Periodic + (void *)sg, + on_ping_interval_expired + ); + if (sg->ping_interval_timer == NULL) { + free(sg); + return SIGNAL_ERR_OTHER; + } + sg->ping_timeout_timer = xTimerCreate( + "ping_timeout", + pdMS_TO_TICKS(1000), // Will be overwritten before start + pdFALSE, // One-shot + (void *)sg, + on_ping_timeout_expired + ); + if (sg->ping_timeout_timer == NULL) { + xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); free(sg); return SIGNAL_ERR_OTHER; } @@ -214,8 +245,8 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) }; sg->ws = esp_websocket_client_init(&ws_config); if (sg->ws == NULL) { - ESP_LOGE(TAG, "Failed to initialize WebSocket client"); - esp_timer_delete(sg->ping_timer); + xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); + xTimerDelete(sg->ping_timeout_timer, portMAX_DELAY); free(sg); return SIGNAL_ERR_WEBSOCKET; } @@ -235,7 +266,8 @@ signal_err_t signal_destroy(signal_handle_t handle) return SIGNAL_ERR_INVALID_ARG; } signal_t *sg = (signal_t *)handle; - esp_timer_delete(sg->ping_timer); + xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); + xTimerDelete(sg->ping_timeout_timer, portMAX_DELAY); signal_close(handle); esp_websocket_client_destroy(sg->ws); free(sg); @@ -273,11 +305,8 @@ signal_err_t signal_close(signal_handle_t handle) return SIGNAL_ERR_INVALID_ARG; } signal_t *sg = (signal_t *)handle; - - esp_timer_stop(sg->ping_timer); if (esp_websocket_client_is_connected(sg->ws) && esp_websocket_client_close(sg->ws, pdMS_TO_TICKS(SIGNAL_WS_CLOSE_TIMEOUT_MS)) != ESP_OK) { - ESP_LOGE(TAG, "Failed to close WebSocket"); return SIGNAL_ERR_WEBSOCKET; } return SIGNAL_ERR_NONE; diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index a69f79c..05a80a5 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -2,7 +2,6 @@ #pragma once #include "protocol.h" -#include "common.h" #ifdef __cplusplus extern "C" { @@ -23,20 +22,20 @@ typedef enum { /// Signal connection state. typedef enum { - /// Unknown state. - SIGNAL_STATE_UNKNOWN = 0, - /// Disconnected. - SIGNAL_STATE_DISCONNECTED = 1 << 0, + SIGNAL_STATE_DISCONNECTED = 0, /// Establishing connection. - SIGNAL_STATE_CONNECTING = 1 << 1, + SIGNAL_STATE_CONNECTING = 1 << 0, /// Connection established. - SIGNAL_STATE_CONNECTED = 1 << 2, + SIGNAL_STATE_CONNECTED = 1 << 1, /// Server unreachable. - SIGNAL_STATE_FAILED_UNREACHABLE = 1 << 3, + SIGNAL_STATE_FAILED_UNREACHABLE = 1 << 2, + + /// Server did not respond to ping within timeout window. + SIGNAL_STATE_FAILED_PING_TIMEOUT = 1 << 3, /// Internal server error. SIGNAL_STATE_FAILED_INTERNAL = 1 << 4, @@ -51,13 +50,14 @@ typedef enum { SIGNAL_STATE_FAILED_CLIENT_OTHER = 1 << 7, /// Any client failure (retry should not be attempted). - SIGNAL_STATE_FAILED_CLIENT_ANY = SIGNAL_STATE_FAILED_BAD_TOKEN | + SIGNAL_STATE_FAILED_CLIENT_ANY = SIGNAL_STATE_FAILED_BAD_TOKEN | SIGNAL_STATE_FAILED_UNAUTHORIZED | SIGNAL_STATE_FAILED_CLIENT_OTHER, /// Any failure. - SIGNAL_STATE_FAILED_ANY = SIGNAL_STATE_FAILED_UNREACHABLE | - SIGNAL_STATE_FAILED_INTERNAL | + SIGNAL_STATE_FAILED_ANY = SIGNAL_STATE_FAILED_UNREACHABLE | + SIGNAL_STATE_FAILED_PING_TIMEOUT | + SIGNAL_STATE_FAILED_INTERNAL | SIGNAL_STATE_FAILED_CLIENT_ANY } signal_state_t; diff --git a/components/livekit/include/livekit_types.h b/components/livekit/include/livekit_types.h index 6650ae6..550ed6c 100644 --- a/components/livekit/include/livekit_types.h +++ b/components/livekit/include/livekit_types.h @@ -56,6 +56,12 @@ typedef enum { /// LIVEKIT_FAILURE_REASON_MAX_RETRIES, + /// Ping Timeout + /// + /// Server did not respond to ping within the timeout window. + /// + LIVEKIT_FAILURE_REASON_PING_TIMEOUT, + /// Duplicate Identity /// /// Another participant already has the same identity. From aefc951f630cb5cbe5b784f7401255c9b384d124 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Sat, 23 Aug 2025 20:57:08 +1000 Subject: [PATCH 52/81] Cleanup signal client init --- components/livekit/core/engine.c | 3 +- components/livekit/core/signaling.c | 53 ++++++++++++++--------------- components/livekit/core/signaling.h | 2 +- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 4c6a970..4e86df2 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1027,7 +1027,8 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) .on_state_changed = on_signal_state_changed, .on_res = on_signal_res, }; - if (signal_create(&eng->signal_handle, &signal_options) != SIGNAL_ERR_NONE) { + eng->signal_handle = signal_init(&signal_options); + if (eng->signal_handle == NULL) { free(eng->event_queue); free(eng->timer); free(eng); diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 3ef2d90..d7fbd9a 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -191,21 +191,16 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void } } -signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) +signal_handle_t signal_init(const signal_options_t *options) { - if (options == NULL || handle == NULL) { - return SIGNAL_ERR_INVALID_ARG; - } - - if (options->on_state_changed == NULL || + if (options == NULL || + options->on_state_changed == NULL || options->on_res == NULL) { - ESP_LOGE(TAG, "Missing required event handlers"); - return SIGNAL_ERR_INVALID_ARG; + return NULL; } - signal_t *sg = calloc(1, sizeof(signal_t)); if (sg == NULL) { - return SIGNAL_ERR_NO_MEM; + return NULL; } sg->options = *options; @@ -217,8 +212,7 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) on_ping_interval_expired ); if (sg->ping_interval_timer == NULL) { - free(sg); - return SIGNAL_ERR_OTHER; + goto _init_failed; } sg->ping_timeout_timer = xTimerCreate( "ping_timeout", @@ -228,11 +222,8 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) on_ping_timeout_expired ); if (sg->ping_timeout_timer == NULL) { - xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); - free(sg); - return SIGNAL_ERR_OTHER; + goto _init_failed; } - // URL will be set on connect static esp_websocket_client_config_t ws_config = { .buffer_size = SIGNAL_WS_BUFFER_SIZE, @@ -245,19 +236,20 @@ signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options) }; sg->ws = esp_websocket_client_init(&ws_config); if (sg->ws == NULL) { - xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); - xTimerDelete(sg->ping_timeout_timer, portMAX_DELAY); - free(sg); - return SIGNAL_ERR_WEBSOCKET; + goto _init_failed; } - esp_websocket_register_events( + if (esp_websocket_register_events( sg->ws, WEBSOCKET_EVENT_ANY, on_ws_event, (void *)sg - ); - *handle = sg; - return SIGNAL_ERR_NONE; + ) != ESP_OK) { + goto _init_failed; + } + return sg; +_init_failed: + signal_destroy(sg); + return NULL; } signal_err_t signal_destroy(signal_handle_t handle) @@ -266,10 +258,15 @@ signal_err_t signal_destroy(signal_handle_t handle) return SIGNAL_ERR_INVALID_ARG; } signal_t *sg = (signal_t *)handle; - xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); - xTimerDelete(sg->ping_timeout_timer, portMAX_DELAY); - signal_close(handle); - esp_websocket_client_destroy(sg->ws); + if (sg->ping_interval_timer != NULL) { + xTimerDelete(sg->ping_interval_timer, portMAX_DELAY); + } + if (sg->ping_timeout_timer != NULL) { + xTimerDelete(sg->ping_timeout_timer, portMAX_DELAY); + } + if (sg->ws != NULL) { + esp_websocket_client_destroy(sg->ws); + } free(sg); return SIGNAL_ERR_NONE; } diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index 05a80a5..13ca7c8 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -76,7 +76,7 @@ typedef struct { bool (*on_res)(livekit_pb_signal_response_t *res, void *ctx); } signal_options_t; -signal_err_t signal_create(signal_handle_t *handle, signal_options_t *options); +signal_handle_t signal_init(const signal_options_t *options); signal_err_t signal_destroy(signal_handle_t handle); /// Establishes the WebSocket connection From c49b36430161c5bf2c47978609ec4571383ad587 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Sat, 23 Aug 2025 21:10:18 +1000 Subject: [PATCH 53/81] Cleanup engine init --- components/livekit/core/engine.c | 105 ++++++++++++++++++------------ components/livekit/core/engine.h | 3 +- components/livekit/core/livekit.c | 3 +- 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 4e86df2..083aff6 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -996,30 +996,44 @@ static void engine_task(void *arg) // MARK: - Public API -engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) +engine_handle_t engine_init(const engine_options_t *options) { engine_t *eng = (engine_t *)calloc(1, sizeof(engine_t)); if (eng == NULL) { - return ENGINE_ERR_NO_MEM; + return NULL; } + eng->options = *options; + eng->state = ENGINE_STATE_DISCONNECTED; + eng->is_running = true; - eng->event_queue = xQueueCreate(CONFIG_LK_ENGINE_QUEUE_SIZE, sizeof(engine_event_t)); + eng->event_queue = xQueueCreate( + CONFIG_LK_ENGINE_QUEUE_SIZE, + sizeof(engine_event_t) + ); if (eng->event_queue == NULL) { - free(eng); - return ENGINE_ERR_NO_MEM; + goto _init_failed; + } + + if (xTaskCreate( + engine_task, + "engine_task", + 4096, + (void *)eng, + 5, + &eng->task_handle + ) != pdPASS) { + goto _init_failed; } eng->timer = xTimerCreate( "lk_engine_timer", pdMS_TO_TICKS(1000), pdFALSE, - eng, - on_timer_expired); - + (void *)eng, + on_timer_expired + ); if (eng->timer == NULL) { - free(eng->event_queue); - free(eng); - return ENGINE_ERR_NO_MEM; + goto _init_failed; } signal_options_t signal_options = { @@ -1029,21 +1043,7 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) }; eng->signal_handle = signal_init(&signal_options); if (eng->signal_handle == NULL) { - free(eng->event_queue); - free(eng->timer); - free(eng); - return ENGINE_ERR_SIGNALING; - } - - eng->options = *options; - eng->state = ENGINE_STATE_DISCONNECTED; - eng->is_running = true; - - if (xTaskCreate(engine_task, "engine_task", 4096, eng, 5, &eng->task_handle) != pdPASS) { - free(eng->event_queue); - free(eng->timer); - free(eng); - return ENGINE_ERR_NO_MEM; + goto _init_failed; } esp_capture_sink_cfg_t sink_cfg = { @@ -1060,16 +1060,29 @@ engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options) .fps = eng->options.media.video_info.fps, }, }; - if (options->media.audio_info.codec != ESP_PEER_AUDIO_CODEC_NONE) { // TODO: Can we ensure the renderer is valid? If not, return error. eng->renderer_handle = options->media.renderer; } - esp_capture_setup_path(eng->options.media.capturer, ESP_CAPTURE_PATH_PRIMARY, &sink_cfg, &eng->capturer_path); - esp_capture_enable_path(eng->capturer_path, ESP_CAPTURE_RUN_TYPE_ALWAYS); + if (esp_capture_setup_path( + eng->options.media.capturer, + ESP_CAPTURE_PATH_PRIMARY, + &sink_cfg, + &eng->capturer_path + ) != ESP_CAPTURE_ERR_OK) { + goto _init_failed; + } + if (esp_capture_enable_path( + eng->capturer_path, + ESP_CAPTURE_RUN_TYPE_ALWAYS + ) != ESP_CAPTURE_ERR_OK) { + goto _init_failed; + } + return eng; - *handle = (engine_handle_t)eng; - return ENGINE_ERR_NONE; +_init_failed: + engine_destroy(eng); + return NULL; } engine_err_t engine_destroy(engine_handle_t handle) @@ -1078,25 +1091,33 @@ engine_err_t engine_destroy(engine_handle_t handle) return ENGINE_ERR_INVALID_ARG; } engine_t *eng = (engine_t *)handle; - eng->is_running = false; if (eng->task_handle != NULL) { // TODO: Wait for disconnected state or timeout vTaskDelay(pdMS_TO_TICKS(100)); + vTaskDelete(eng->task_handle); + } + if (eng->timer != NULL) { + xTimerDelete(eng->timer, portMAX_DELAY); + } + if (eng->event_queue != NULL) { + vQueueDelete(eng->event_queue); + } + if (eng->signal_handle != NULL) { + signal_destroy(eng->signal_handle); } - vTaskDelete(eng->task_handle); - - xTimerDelete(eng->timer, portMAX_DELAY); - vQueueDelete(eng->event_queue); - - signal_destroy(eng->signal_handle); if (eng->pub_peer_handle != NULL) { peer_destroy(eng->pub_peer_handle); } - - if (eng->server_url != NULL) free(eng->server_url); - if (eng->token != NULL) free(eng->token); - // TODO: Free other resources + if (eng->sub_peer_handle != NULL) { + peer_destroy(eng->sub_peer_handle); + } + if (eng->server_url != NULL) { + free(eng->server_url); + } + if (eng->token != NULL) { + free(eng->token); + } free(eng); return ENGINE_ERR_NONE; } diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 4223145..823ebd9 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -48,8 +48,7 @@ typedef struct { } engine_options_t; /// Creates a new instance. -/// @param[out] handle The handle to the new instance. -engine_err_t engine_create(engine_handle_t *handle, engine_options_t *options); +engine_handle_t engine_init(const engine_options_t *options); /// Destroys an instance. /// @param[in] handle The handle to the instance to destroy. diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index c190ae3..0121901 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -215,7 +215,8 @@ livekit_err_t livekit_room_create(livekit_room_handle_t *handle, const livekit_r int ret = LIVEKIT_ERR_OTHER; do { - if (engine_create(&room->engine, &eng_options) != ENGINE_ERR_NONE) { + room->engine = engine_init(&eng_options); + if (room->engine == NULL) { ESP_LOGE(TAG, "Failed to create engine"); ret = LIVEKIT_ERR_ENGINE; break; From f5858161680e42d0bab10e4fb5e7ff49363ac700 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:07:15 +1000 Subject: [PATCH 54/81] Use protocol helper for encoding requests --- components/livekit/core/protocol.c | 24 +++++++++++++++++++++++- components/livekit/core/protocol.h | 15 +++++++++++++++ components/livekit/core/signaling.c | 13 ++++--------- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index f400627..7ffe1db 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -84,4 +84,26 @@ bool protocol_signal_trickle_get_candidate(const livekit_pb_trickle_request_t *t cJSON_Delete(candidate_init); return ret; -} \ No newline at end of file +} + +__attribute__((always_inline)) +inline size_t protocol_signal_request_encoded_size(const livekit_pb_signal_request_t *req) +{ + size_t encoded_size = 0; + if (!pb_get_encoded_size(&encoded_size, LIVEKIT_PB_SIGNAL_REQUEST_FIELDS, req)) { + return 0; + } + return encoded_size; +} + +__attribute__((always_inline)) +inline bool protocol_signal_request_encode(const livekit_pb_signal_request_t *req, uint8_t *dest, size_t encoded_size) +{ + pb_ostream_t stream = pb_ostream_from_buffer((pb_byte_t *)dest, encoded_size); + if (!pb_encode(&stream, LIVEKIT_PB_SIGNAL_REQUEST_FIELDS, req)) { + ESP_LOGE(TAG, "Failed to encode signal req: type=%" PRIu16 ", error=%s", + req->which_message, stream.errmsg); + return false; + } + return stream.bytes_written == encoded_size; +} diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 8071fc7..a2ace1d 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -16,6 +16,8 @@ extern "C" { /// Server identifier (SID) type. typedef char livekit_pb_sid_t[16]; +// MARK: - Data Packet + /// Decodes a data packet. /// /// When the packet is no longer needed, free using `protocol_data_packet_free`. @@ -25,6 +27,8 @@ bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data /// Frees a data packet. void protocol_data_packet_free(livekit_pb_data_packet_t *packet); +// MARK: - Signal Response + /// Decodes a signal response. /// /// When the response is no longer needed, free using `protocol_signal_res_free`. @@ -43,6 +47,17 @@ bool protocol_signal_trickle_get_candidate( char **candidate_out ); +// MARK: - Signal Request + +/// Returns the encoded size of a signal request. +/// +/// @returns The encoded size of the request or 0 if the encoded size cannot be determined. +/// +size_t protocol_signal_request_encoded_size(const livekit_pb_signal_request_t *req); + +/// Encodes a signal request into the provided buffer. +bool protocol_signal_request_encode(const livekit_pb_signal_request_t *req, uint8_t *dest, size_t encoded_size); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index d7fbd9a..7d97bca 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -50,11 +50,8 @@ static inline signal_state_t failed_state_from_http_status(int status) static signal_err_t send_request(signal_t *sg, livekit_pb_signal_request_t *request) { - // TODO: Optimize (use static buffer for small messages) - ESP_LOGD(TAG, "Sending request: type=%d", request->which_message); - - size_t encoded_size = 0; - if (!pb_get_encoded_size(&encoded_size, LIVEKIT_PB_SIGNAL_REQUEST_FIELDS, request)) { + size_t encoded_size = protocol_signal_request_encoded_size(request); + if (encoded_size == 0) { return SIGNAL_ERR_MESSAGE; } uint8_t *enc_buf = (uint8_t *)malloc(encoded_size); @@ -63,15 +60,13 @@ static signal_err_t send_request(signal_t *sg, livekit_pb_signal_request_t *requ } int ret = SIGNAL_ERR_NONE; do { - pb_ostream_t stream = pb_ostream_from_buffer(enc_buf, encoded_size); - if (!pb_encode(&stream, LIVEKIT_PB_SIGNAL_REQUEST_FIELDS, request)) { - ESP_LOGE(TAG, "Failed to encode request"); + if (!protocol_signal_request_encode(request, enc_buf, encoded_size)) { ret = SIGNAL_ERR_MESSAGE; break; } if (esp_websocket_client_send_bin(sg->ws, (const char *)enc_buf, - stream.bytes_written, + encoded_size, portMAX_DELAY) < 0) { ESP_LOGE(TAG, "Failed to send request"); ret = SIGNAL_ERR_MESSAGE; From a595f093e3708dee3826b6f2507eb1d5d530513a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:14:56 +1000 Subject: [PATCH 55/81] Use protocol helpers for encoding data packets --- components/livekit/core/peer.c | 13 ++++--------- components/livekit/core/protocol.c | 28 ++++++++++++++++++++++++++++ components/livekit/core/protocol.h | 15 ++++++++++++--- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index 1241a74..e0c3af6 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -434,27 +434,22 @@ peer_err_t peer_send_data_packet(peer_handle_t handle, const livekit_pb_data_pac .stream_id = stream_id }; - // TODO: Optimize encoding - size_t encoded_size = 0; - if (!pb_get_encoded_size(&encoded_size, LIVEKIT_PB_DATA_PACKET_FIELDS, packet)) { + size_t encoded_size = protocol_data_packet_encoded_size(packet); + if (encoded_size == 0) { return PEER_ERR_MESSAGE; } uint8_t *enc_buf = (uint8_t *)malloc(encoded_size); if (enc_buf == NULL) { return PEER_ERR_NO_MEM; } - int ret = PEER_ERR_NONE; do { - pb_ostream_t stream = pb_ostream_from_buffer(enc_buf, encoded_size); - if (!pb_encode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, packet)) { - ESP_LOGE(TAG(peer), "Failed to encode data packet"); + if (!protocol_data_packet_encode(packet, enc_buf, encoded_size)) { ret = PEER_ERR_MESSAGE; break; } - frame_info.data = enc_buf; - frame_info.size = stream.bytes_written; + frame_info.size = encoded_size; if (esp_peer_send_data(peer->connection, &frame_info) != ESP_PEER_ERR_NONE) { ESP_LOGE(TAG(peer), "Data channel send failed"); ret = PEER_ERR_RTC; diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index 7ffe1db..226521b 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -17,6 +17,8 @@ static int32_t decode_first_tag(const pb_byte_t *buf, size_t len) return (int32_t)tag; } +// MARK: - Data packet + bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); @@ -33,6 +35,30 @@ void protocol_data_packet_free(livekit_pb_data_packet_t *packet) pb_release(LIVEKIT_PB_DATA_PACKET_FIELDS, packet); } +__attribute__((always_inline)) +inline size_t protocol_data_packet_encoded_size(const livekit_pb_data_packet_t *packet) +{ + size_t encoded_size = 0; + if (!pb_get_encoded_size(&encoded_size, LIVEKIT_PB_DATA_PACKET_FIELDS, packet)) { + return 0; + } + return encoded_size; +} + +__attribute__((always_inline)) +inline bool protocol_data_packet_encode(const livekit_pb_data_packet_t *packet, uint8_t *dest, size_t encoded_size) +{ + pb_ostream_t stream = pb_ostream_from_buffer((pb_byte_t *)dest, encoded_size); + if (!pb_encode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, packet)) { + ESP_LOGE(TAG, "Failed to encode data packet: type=%" PRIu16 ", error=%s", + packet->which_value, stream.errmsg); + return false; + } + return stream.bytes_written == encoded_size; +} + +// MARK: - Signal response + bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); @@ -86,6 +112,8 @@ bool protocol_signal_trickle_get_candidate(const livekit_pb_trickle_request_t *t return ret; } +// MARK: - Signal request + __attribute__((always_inline)) inline size_t protocol_signal_request_encoded_size(const livekit_pb_signal_request_t *req) { diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index a2ace1d..8165806 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -16,7 +16,7 @@ extern "C" { /// Server identifier (SID) type. typedef char livekit_pb_sid_t[16]; -// MARK: - Data Packet +// MARK: - Data packet /// Decodes a data packet. /// @@ -27,7 +27,16 @@ bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data /// Frees a data packet. void protocol_data_packet_free(livekit_pb_data_packet_t *packet); -// MARK: - Signal Response +/// Returns the encoded size of a data packet. +/// +/// @returns The encoded size of the packet or 0 if the encoded size cannot be determined. +/// +size_t protocol_data_packet_encoded_size(const livekit_pb_data_packet_t *packet); + +/// Encodes a data packet into the provided buffer. +bool protocol_data_packet_encode(const livekit_pb_data_packet_t *packet, uint8_t *dest, size_t encoded_size); + +// MARK: - Signal response /// Decodes a signal response. /// @@ -47,7 +56,7 @@ bool protocol_signal_trickle_get_candidate( char **candidate_out ); -// MARK: - Signal Request +// MARK: - Signal request /// Returns the encoded size of a signal request. /// From c996bad27a239a493c18daba4e4e15d163d85063 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:18:40 +1000 Subject: [PATCH 56/81] Inline remaining protocol helpers --- components/livekit/core/protocol.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index 226521b..dfc2dd5 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -19,7 +19,8 @@ static int32_t decode_first_tag(const pb_byte_t *buf, size_t len) // MARK: - Data packet -bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out) +__attribute__((always_inline)) +inline bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data_packet_t *out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_DATA_PACKET_FIELDS, out)) { @@ -30,7 +31,8 @@ bool protocol_data_packet_decode(const uint8_t *buf, size_t len, livekit_pb_data return true; } -void protocol_data_packet_free(livekit_pb_data_packet_t *packet) +__attribute__((always_inline)) +inline void protocol_data_packet_free(livekit_pb_data_packet_t *packet) { pb_release(LIVEKIT_PB_DATA_PACKET_FIELDS, packet); } @@ -59,7 +61,8 @@ inline bool protocol_data_packet_encode(const livekit_pb_data_packet_t *packet, // MARK: - Signal response -bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) +__attribute__((always_inline)) +inline bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, out)) { @@ -70,7 +73,8 @@ bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signa return true; } -void protocol_signal_res_free(livekit_pb_signal_response_t *res) +__attribute__((always_inline)) +inline void protocol_signal_res_free(livekit_pb_signal_response_t *res) { pb_release(LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, res); } From b062bdd90e3e3cb9c9e72f9bfe99e042f20f8339 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:21:25 +1000 Subject: [PATCH 57/81] Use consistent naming convention --- components/livekit/core/engine.c | 2 +- components/livekit/core/protocol.c | 4 ++-- components/livekit/core/protocol.h | 6 +++--- components/livekit/core/signaling.c | 6 +++--- components/livekit/core/signaling.h | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 083aff6..d8e539f 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -579,7 +579,7 @@ static void event_free(engine_event_t *ev) protocol_data_packet_free(&ev->detail.data_packet); break; case EV_SIG_RES: - protocol_signal_res_free(&ev->detail.res); + protocol_signal_response_free(&ev->detail.res); break; default: break; } diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index dfc2dd5..0fa8241 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -62,7 +62,7 @@ inline bool protocol_data_packet_encode(const livekit_pb_data_packet_t *packet, // MARK: - Signal response __attribute__((always_inline)) -inline bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) +inline bool protocol_signal_response_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out) { pb_istream_t stream = pb_istream_from_buffer((const pb_byte_t *)buf, len); if (!pb_decode(&stream, LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, out)) { @@ -74,7 +74,7 @@ inline bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_p } __attribute__((always_inline)) -inline void protocol_signal_res_free(livekit_pb_signal_response_t *res) +inline void protocol_signal_response_free(livekit_pb_signal_response_t *res) { pb_release(LIVEKIT_PB_SIGNAL_RESPONSE_FIELDS, res); } diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 8165806..367eb6c 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -40,12 +40,12 @@ bool protocol_data_packet_encode(const livekit_pb_data_packet_t *packet, uint8_t /// Decodes a signal response. /// -/// When the response is no longer needed, free using `protocol_signal_res_free`. +/// When the response is no longer needed, free using `protocol_signal_response_free`. /// -bool protocol_signal_res_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out); +bool protocol_signal_response_decode(const uint8_t *buf, size_t len, livekit_pb_signal_response_t* out); /// Frees a signal response. -void protocol_signal_res_free(livekit_pb_signal_response_t *res); +void protocol_signal_response_free(livekit_pb_signal_response_t *res); /// Extract ICE candidate string from a trickle request. /// diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 7d97bca..80aa2e6 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -168,17 +168,17 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void } if (data->data_len < 1) break; livekit_pb_signal_response_t res = {}; - if (!protocol_signal_res_decode((const uint8_t *)data->data_ptr, data->data_len, &res)) { + if (!protocol_signal_response_decode((const uint8_t *)data->data_ptr, data->data_len, &res)) { break; } if (!res_middleware(sg, &res)) { // Don't forward. - protocol_signal_res_free(&res); + protocol_signal_response_free(&res); break; } if (!sg->options.on_res(&res, sg->options.ctx)) { // Ownership was not taken. - protocol_signal_res_free(&res); + protocol_signal_response_free(&res); } break; default: diff --git a/components/livekit/core/signaling.h b/components/livekit/core/signaling.h index 13ca7c8..e967c6b 100644 --- a/components/livekit/core/signaling.h +++ b/components/livekit/core/signaling.h @@ -71,7 +71,7 @@ typedef struct { /// /// The receiver returns true to take ownership of the response. If /// ownership is not taken (false), the response will be freed with - /// `protocol_signal_res_free` internally. + /// `protocol_signal_response_free` internally. /// bool (*on_res)(livekit_pb_signal_response_t *res, void *ctx); } signal_options_t; From 713733095c0059063105f8d3ce28335dde6d8049 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:22:27 +1000 Subject: [PATCH 58/81] Make nanopb encode/decode includes private --- components/livekit/core/protocol.c | 3 +++ components/livekit/core/protocol.h | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c index 0fa8241..607b2c6 100644 --- a/components/livekit/core/protocol.c +++ b/components/livekit/core/protocol.c @@ -1,6 +1,9 @@ #include #include "esp_log.h" #include "cJSON.h" +#include "pb_encode.h" +#include "pb_decode.h" + #include "protocol.h" static const char *TAG = "livekit_protocol"; diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 367eb6c..d4d675a 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -1,9 +1,6 @@ #pragma once -#include "pb_encode.h" -#include "pb_decode.h" - #include "livekit_rtc.pb.h" #include "livekit_models.pb.h" #include "livekit_metrics.pb.h" From 78657157c6c9a31d88e531846a9f1e3eb4eb8712 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:11:46 +1000 Subject: [PATCH 59/81] Simplify internal API --- components/livekit/core/engine.c | 4 ++-- components/livekit/core/engine.h | 2 +- components/livekit/core/livekit.c | 10 ++-------- components/livekit/core/peer.c | 4 ++-- components/livekit/core/peer.h | 2 +- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index d8e539f..d89e7a3 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1163,12 +1163,12 @@ livekit_failure_reason_t engine_get_failure_reason(engine_handle_t handle) return eng->failure_reason; } -engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind) +engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, bool reliable) { if (handle == NULL) { return ENGINE_ERR_INVALID_ARG; } - // engine_t *eng = (engine_t *)handle; + engine_t *eng = (engine_t *)handle; // TODO: Send data packet return ENGINE_ERR_NONE; diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index 823ebd9..f5c8ecd 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -64,7 +64,7 @@ engine_err_t engine_close(engine_handle_t handle); livekit_failure_reason_t engine_get_failure_reason(engine_handle_t handle); /// Sends a data packet to the remote peer. -engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind); +engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_data_packet_t* packet, bool reliable); #ifdef __cplusplus } diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index 0121901..8e235d7 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -18,10 +18,7 @@ typedef struct { static bool send_reliable_packet(const livekit_pb_data_packet_t* packet, void *ctx) { livekit_room_t *room = (livekit_room_t *)ctx; - return engine_send_data_packet( - room->engine, - packet, - LIVEKIT_PB_DATA_PACKET_KIND_RELIABLE) == ENGINE_ERR_NONE; + return engine_send_data_packet(room->engine, packet, true) == ENGINE_ERR_NONE; } static void on_rpc_result(const livekit_rpc_result_t* result, void* ctx) @@ -360,10 +357,7 @@ livekit_err_t livekit_room_publish_data(livekit_room_handle_t handle, livekit_da packet.destination_identities = options->destination_identities; // TODO: Set sender identity - livekit_pb_data_packet_kind_t kind = options->lossy ? - LIVEKIT_PB_DATA_PACKET_KIND_LOSSY : LIVEKIT_PB_DATA_PACKET_KIND_RELIABLE; - - if (engine_send_data_packet(room->engine, &packet, kind) != ENGINE_ERR_NONE) { + if (engine_send_data_packet(room->engine, &packet, !options->lossy) != ENGINE_ERR_NONE) { ESP_LOGE(TAG, "Failed to send data packet"); free(bytes_array); return LIVEKIT_ERR_ENGINE; diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index e0c3af6..a283756 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -416,14 +416,14 @@ peer_err_t peer_handle_ice_candidate(peer_handle_t handle, const char *candidate return PEER_ERR_NONE; } -peer_err_t peer_send_data_packet(peer_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind) +peer_err_t peer_send_data_packet(peer_handle_t handle, const livekit_pb_data_packet_t* packet, bool reliable) { if (handle == NULL || packet == NULL) { return PEER_ERR_INVALID_ARG; } peer_t *peer = (peer_t *)handle; - uint16_t stream_id = kind == LIVEKIT_PB_DATA_PACKET_KIND_RELIABLE ? + uint16_t stream_id = reliable ? peer->reliable_stream_id : peer->lossy_stream_id; if (stream_id == STREAM_ID_INVALID) { ESP_LOGE(TAG(peer), "Required data channel not connected"); diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index 23c4c70..8aefab2 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -90,7 +90,7 @@ peer_err_t peer_handle_sdp(peer_handle_t handle, const char *sdp); peer_err_t peer_handle_ice_candidate(peer_handle_t handle, const char *candidate); /// Sends a data packet to the remote peer. -peer_err_t peer_send_data_packet(peer_handle_t handle, const livekit_pb_data_packet_t* packet, livekit_pb_data_packet_kind_t kind); +peer_err_t peer_send_data_packet(peer_handle_t handle, const livekit_pb_data_packet_t* packet, bool reliable); /// Sends an audio frame to the remote peer. /// @warning Only use on publisher peer. From b34f9aa212724e107a8b663af6591956f50223d9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:33:57 +1000 Subject: [PATCH 60/81] Send data packets --- components/livekit/core/engine.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index d89e7a3..70af692 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1169,7 +1169,13 @@ engine_err_t engine_send_data_packet(engine_handle_t handle, const livekit_pb_da return ENGINE_ERR_INVALID_ARG; } engine_t *eng = (engine_t *)handle; - // TODO: Send data packet - + // TODO: Implement buffering for reliable packets + if (eng->state != ENGINE_STATE_CONNECTED) { + return ENGINE_ERR_OTHER; + } + if (eng->pub_peer_handle == NULL || + peer_send_data_packet(eng->pub_peer_handle, packet, reliable) != PEER_ERR_NONE) { + return ENGINE_ERR_RTC; + } return ENGINE_ERR_NONE; } \ No newline at end of file From efd71e912621f33872931e864b74ca91deb500db Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 26 Aug 2025 10:14:56 +1000 Subject: [PATCH 61/81] Zero only --- components/livekit/core/signaling.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 80aa2e6..7e8137b 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -68,7 +68,7 @@ static signal_err_t send_request(signal_t *sg, livekit_pb_signal_request_t *requ (const char *)enc_buf, encoded_size, portMAX_DELAY) < 0) { - ESP_LOGE(TAG, "Failed to send request"); + //ESP_LOGE(TAG, "Failed to send request"); ret = SIGNAL_ERR_MESSAGE; break; } @@ -81,7 +81,7 @@ static void on_ping_interval_expired(TimerHandle_t handle) { signal_t *sg = (signal_t *)pvTimerGetTimerID(handle); - livekit_pb_signal_request_t req = LIVEKIT_PB_SIGNAL_REQUEST_INIT_DEFAULT; + livekit_pb_signal_request_t req = {}; req.which_message = LIVEKIT_PB_SIGNAL_REQUEST_PING_REQ_TAG; req.message.ping_req.timestamp = get_unix_time_ms(); req.message.ping_req.rtt = sg->rtt; From c63c7c50f37e16417ea76fdd4352fa5d3a19f108 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:01:27 +1000 Subject: [PATCH 62/81] Remove state guards for sub audio Audio stream can begin while still in connecting state --- components/livekit/core/engine.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 70af692..f80d52e 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -142,6 +142,7 @@ static engine_err_t subscribe_tracks(engine_t *eng, livekit_pb_track_info_t *tra continue; } // For now, subscribe to the first audio track. + ESP_LOGI(TAG, "Subscribing to audio track: sid=%s", track->sid); signal_send_update_subscription(eng->signal_handle, track->sid, true); strncpy(eng->session.sub_audio_track_sid, track->sid, sizeof(eng->session.sub_audio_track_sid)); break; @@ -400,7 +401,6 @@ static void on_peer_sub_answer(const char *sdp, void *ctx) static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx) { engine_t *eng = (engine_t *)ctx; - if (eng->state != ENGINE_STATE_CONNECTED) return; av_render_audio_info_t render_info = {}; convert_dec_aud_info(info, &render_info); @@ -416,8 +416,6 @@ static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) { engine_t *eng = (engine_t *)ctx; - if (eng->state != ENGINE_STATE_CONNECTED) return; - av_render_audio_data_t audio_data = { .pts = frame->pts, .data = frame->data, From 21538756b43571f5c8faedf4a9af478840ab4b26 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:04:23 +1000 Subject: [PATCH 63/81] Handle room and participant update in connecting state Early participant update messages are necessary to subscribe to remote track --- components/livekit/core/engine.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index f80d52e..c068ab0 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -743,6 +743,14 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) eng->failure_reason = map_disconnect_reason(leave->reason); eng->state = ENGINE_STATE_DISCONNECTED; break; + case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: + livekit_pb_room_update_t *room_update = &res->message.room_update; + handle_room_update(eng, room_update); + break; + case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: + livekit_pb_participant_update_t *update = &res->message.update; + handle_participant_update(eng, update); + break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: livekit_pb_join_response_t *join = &res->message.join; handle_join(eng, join); From 0f774a1a9ddaabff143cd01a27c42660a310c08e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:22:05 +1000 Subject: [PATCH 64/81] Cleanup previous connection in backoff Perform same cleanup steps when entering both disconnected and backoff --- components/livekit/core/engine.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index c068ab0..46c71ae 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -690,6 +690,15 @@ static void handle_participant_update(engine_t *eng, livekit_pb_participant_upda } } +/// Cleans up resources and state from the previous connection. +static void cleanup_previous_connection(engine_t *eng) +{ + media_stream_end(eng); + signal_close(eng->signal_handle); + destroy_peer_connections(eng); + memset(&eng->session, 0, sizeof(eng->session)); +} + // MARK: - State: Disconnected /// Handler for `ENGINE_STATE_DISCONNECTED`. @@ -697,12 +706,7 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: - // Clean up resources from previous connection (if any) - media_stream_end(eng); - signal_close(eng->signal_handle); - destroy_peer_connections(eng); - - memset(&eng->session, 0, sizeof(eng->session)); + cleanup_previous_connection(eng); eng->retry_count = 0; break; case EV_CMD_CONNECT: @@ -906,9 +910,7 @@ static bool handle_state_backoff(engine_t *eng, const engine_event_t *ev) { switch (ev->type) { case _EV_STATE_ENTER: - media_stream_end(eng); - signal_close(eng->signal_handle); - destroy_peer_connections(eng); + cleanup_previous_connection(eng); eng->retry_count++; if (eng->retry_count > CONFIG_LK_MAX_RETRIES) { From 5d2dac5a36bd7e5394e01e040d898bf1107f2ddf Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 26 Aug 2025 17:52:15 +1000 Subject: [PATCH 65/81] Handle ICE servers Reintroduce support for dynamically setting ICE servers, use hardcoded for now. --- components/livekit/Kconfig | 3 ++ components/livekit/core/engine.c | 85 +++++++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/components/livekit/Kconfig b/components/livekit/Kconfig index 676c717..59dbd21 100644 --- a/components/livekit/Kconfig +++ b/components/livekit/Kconfig @@ -3,6 +3,9 @@ menu "LiveKit" int "Maximum connection retries" range 0 100 default 7 + config LK_MAX_ICE_SERVERS + int "Maximum number of ICE servers" + default 3 config LK_ENGINE_QUEUE_SIZE int "Number of engine events to queue" default 32 diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 46c71ae..2de174c 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -70,7 +70,6 @@ typedef struct { typedef struct { bool is_subscriber_primary; - bool force_relay; livekit_pb_sid_t local_participant_sid; livekit_pb_sid_t sub_audio_track_sid; } session_state_t; @@ -459,11 +458,69 @@ static void destroy_peer_connections(engine_t *eng) _disconnect_and_destroy_peer(&eng->sub_peer_handle); } -static bool establish_peer_connections(engine_t *eng) +/// Maps list of `livekit_pb_ice_server_t` to list of `esp_peer_ice_server_cfg_t`. +/// +/// Note: +/// - A single `livekit_pb_ice_server_t` can contain multiple URLs, which +/// will map to multiple `esp_peer_ice_server_cfg_t` entries. +/// - Strings are not copied, so the caller must ensure the original ICE +/// server list stays alive until the peers are created. +/// +static inline size_t map_ice_servers( + livekit_pb_ice_server_t *pb_servers_list, + int pb_servers_count, + esp_peer_ice_server_cfg_t *server_list, + size_t server_list_capacity +) { + if (pb_servers_list == NULL || + server_list == NULL || + server_list_capacity == 0) { + return 0; + } + size_t count = 0; + for (int i = 0; i < pb_servers_count; i++) { + for (int j = 0; j < pb_servers_list[i].urls_count; j++) { + if (count >= server_list_capacity) { + ESP_LOGW(TAG, "ICE server list capacity exceeded"); + return count; + } + server_list[count].stun_url = pb_servers_list[i].urls[j]; + server_list[count].user = pb_servers_list[i].username; + server_list[count].psw = pb_servers_list[i].credential; + count++; + } + } + return count; +} + +static bool establish_peer_connections(engine_t *eng, livekit_pb_join_response_t *join) { + esp_peer_ice_server_cfg_t server_list[] = { + { .stun_url = "stun:stun.l.google.com:19302" }, + { .stun_url = "stun:stun1.l.google.com:19302" }, + { .stun_url = "stun:stun2.l.google.com:19302" } + }; + int server_count = sizeof(server_list) / sizeof(server_list[0]); + + // TODO: Replace the above with the following to set the ICE servers dynamically: + // esp_peer_ice_server_cfg_t server_list[CONFIG_LK_MAX_ICE_SERVERS]; + // int server_count = map_ice_servers( + // join->ice_servers, + // join->ice_servers_count, + // server_list, + // sizeof(server_list) / sizeof(server_list[0]) + // ); + // if (server_count < 1) { + // ESP_LOGW(TAG, "No ICE servers available"); + // return false; + // } + peer_options_t options = { - .force_relay = eng->session.force_relay, + .force_relay = join->client_configuration.force_relay + == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED, .media = &eng->options.media, + .server_list = server_list, + .server_count = server_count, .on_state_changed = on_peer_state_changed, .on_data_packet = on_peer_data_packet, .on_ice_candidate = on_peer_ice_candidate, @@ -620,14 +677,10 @@ static inline void timer_stop(engine_t *eng) xTimerStop(eng->timer, 0); } -static void handle_join(engine_t *eng, livekit_pb_join_response_t *join) +static bool handle_join(engine_t *eng, livekit_pb_join_response_t *join) { // 1. Store connection settings eng->session.is_subscriber_primary = join->subscriber_primary; - if (join->has_client_configuration) { - eng->session.force_relay = join->client_configuration.force_relay - == LIVEKIT_PB_CLIENT_CONFIG_SETTING_ENABLED; - } // 2. Store local Participant SID strncpy( @@ -648,6 +701,13 @@ static void handle_join(engine_t *eng, livekit_pb_join_response_t *join) eng->options.on_participant_info(&join->other_participants[i], false, eng->options.ctx); } } + + // 5. Establish peer connections + if (!establish_peer_connections(eng, join)) { + ESP_LOGE(TAG, "Failed to establish peer connections"); + return false; + } + return true; } static void handle_trickle(engine_t *eng, livekit_pb_trickle_request_t *trickle) @@ -757,11 +817,8 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) break; case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: livekit_pb_join_response_t *join = &res->message.join; - handle_join(eng, join); - if (!establish_peer_connections(eng)) { - ESP_LOGE(TAG, "Failed to establish peer connections"); - eng->state = ENGINE_STATE_DISCONNECTED; - break; + if (!handle_join(eng, join)) { + eng->state = ENGINE_STATE_BACKOFF; } break; case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: @@ -893,6 +950,8 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) // If either peer fail or disconnects, transition to backoff if (peer_state == CONNECTION_STATE_DISCONNECTED || peer_state == CONNECTION_STATE_FAILED) { + ESP_LOGE(TAG, "%s peer connection failed", + role == PEER_ROLE_PUBLISHER ? "Publisher" : "Subscriber"); eng->failure_reason = LIVEKIT_FAILURE_REASON_RTC; eng->state = ENGINE_STATE_BACKOFF; } From 04f5919d2659a9cb4eb8731d79a13f1d6b21f4d5 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 09:43:49 +1000 Subject: [PATCH 66/81] Use SAFE_FREE macro --- components/livekit/core/engine.c | 18 ++++++------------ components/livekit/core/utils.h | 2 ++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 2de174c..4021906 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -625,10 +625,8 @@ static void event_free(engine_event_t *ev) if (ev == NULL) return; switch (ev->type) { case EV_CMD_CONNECT: - if (ev->detail.cmd_connect.server_url != NULL) - free(ev->detail.cmd_connect.server_url); - if (ev->detail.cmd_connect.token != NULL) - free(ev->detail.cmd_connect.token); + SAFE_FREE(ev->detail.cmd_connect.server_url); + SAFE_FREE(ev->detail.cmd_connect.token); break; case EV_PEER_DATA_PACKET: protocol_data_packet_free(&ev->detail.data_packet); @@ -770,8 +768,8 @@ static bool handle_state_disconnected(engine_t *eng, const engine_event_t *ev) eng->retry_count = 0; break; case EV_CMD_CONNECT: - if (eng->server_url != NULL) free(eng->server_url); - if (eng->token != NULL) free(eng->token); + SAFE_FREE(eng->server_url); + SAFE_FREE(eng->token); eng->server_url = ev->detail.cmd_connect.server_url; eng->token = ev->detail.cmd_connect.token; eng->failure_reason = LIVEKIT_FAILURE_REASON_NONE; @@ -1179,12 +1177,8 @@ engine_err_t engine_destroy(engine_handle_t handle) if (eng->sub_peer_handle != NULL) { peer_destroy(eng->sub_peer_handle); } - if (eng->server_url != NULL) { - free(eng->server_url); - } - if (eng->token != NULL) { - free(eng->token); - } + SAFE_FREE(eng->server_url); + SAFE_FREE(eng->token); free(eng); return ENGINE_ERR_NONE; } diff --git a/components/livekit/core/utils.h b/components/livekit/core/utils.h index ec96d9d..bcacf2b 100644 --- a/components/livekit/core/utils.h +++ b/components/livekit/core/utils.h @@ -7,6 +7,8 @@ extern "C" { #endif +#define SAFE_FREE(ptr) if (ptr != NULL) { free(ptr); ptr = NULL; } + int64_t get_unix_time_ms(void); /// Returns the backoff time in milliseconds for the given attempt number. From 5927628c57d81f3d4584064567e6d958b1bed501 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 10:57:41 +1000 Subject: [PATCH 67/81] Remove dead code --- components/livekit/core/engine.c | 6 ------ components/livekit/core/peer.c | 5 ----- components/livekit/core/peer.h | 3 --- 3 files changed, 14 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 4021906..75ab312 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -376,11 +376,6 @@ static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) return event_enqueue(eng, &ev, false); } -static void on_peer_ice_candidate(const char *candidate, void *ctx) -{ - // TODO: Handle ICE candidate -} - // MARK: - Publisher peer event handlers static void on_peer_pub_offer(const char *sdp, void *ctx) @@ -523,7 +518,6 @@ static bool establish_peer_connections(engine_t *eng, livekit_pb_join_response_t .server_count = server_count, .on_state_changed = on_peer_state_changed, .on_data_packet = on_peer_data_packet, - .on_ice_candidate = on_peer_ice_candidate, .ctx = eng }; diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index a283756..ab99a7d 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -135,10 +135,6 @@ static int on_msg(esp_peer_msg_t *info, void *ctx) (char *)info->data); peer->options.on_sdp((char *)info->data, peer->options.ctx); break; - case ESP_PEER_MSG_TYPE_CANDIDATE: - ESP_LOGI(TAG(peer), "Generated candidate: %s", (char *)info->data); - peer->options.on_ice_candidate((char *)info->data, peer->options.ctx); - break; default: ESP_LOGD(TAG(peer), "Unhandled msg type: %d", info->type); break; @@ -238,7 +234,6 @@ peer_err_t peer_create(peer_handle_t *handle, peer_options_t *options) { if (handle == NULL || options->on_state_changed == NULL || - options->on_ice_candidate == NULL || options->on_sdp == NULL) { return PEER_ERR_INVALID_ARG; } diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index 8aefab2..bebac6b 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -58,9 +58,6 @@ typedef struct { /// an offer or answer depending on target configuration. void (*on_sdp)(const char *sdp, void *ctx); - /// Invoked when a new ICE candidate is available. - void (*on_ice_candidate)(const char *candidate, void *ctx); - /// Invoked when information about an incoming audio stream is available. void (*on_audio_info)(esp_peer_audio_stream_info_t* info, void *ctx); From 891f727b35d7cebc6b591146751df610808cd4f6 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:17:54 +1000 Subject: [PATCH 68/81] Do not handle data packets in FSM --- components/livekit/core/engine.c | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 75ab312..5b21a94 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -33,7 +33,6 @@ typedef enum { EV_SIG_STATE, /// Signal state changed. EV_SIG_RES, /// Signal response received. EV_PEER_STATE, /// Peer state changed. - EV_PEER_DATA_PACKET, /// Peer received data packet. EV_TIMER_EXP, /// Timer expired. EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. _EV_STATE_ENTER, /// State enter hook (internal). @@ -54,9 +53,6 @@ typedef struct { /// Detail for `EV_SIG_RES`. livekit_pb_signal_response_t res; - /// Detail for `EV_PEER_DATA_PACKET`. - livekit_pb_data_packet_t data_packet; - /// Detail for `EV_SIG_STATE`. signal_state_t sig_state; @@ -367,13 +363,11 @@ static void on_peer_state_changed(connection_state_t state, peer_role_t role, vo static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) { engine_t *eng = (engine_t *)ctx; - engine_event_t ev = { - .type = EV_PEER_DATA_PACKET, - .detail.data_packet = *packet - }; - // Returning true indicates ownership of the packet; it will be freed when - // the queue is processed or flushed. - return event_enqueue(eng, &ev, false); + // TODO: Implement buffering for incoming data packets + if (eng->options.on_data_packet) { + eng->options.on_data_packet(packet, eng->options.ctx); + } + return false; } // MARK: - Publisher peer event handlers @@ -622,9 +616,6 @@ static void event_free(engine_event_t *ev) SAFE_FREE(ev->detail.cmd_connect.server_url); SAFE_FREE(ev->detail.cmd_connect.token); break; - case EV_PEER_DATA_PACKET: - protocol_data_packet_free(&ev->detail.data_packet); - break; case EV_SIG_RES: protocol_signal_response_free(&ev->detail.res); break; @@ -884,12 +875,6 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) case EV_CMD_CONNECT: ESP_LOGW(TAG, "Engine already connected, ignoring connect command"); break; - case EV_PEER_DATA_PACKET: - livekit_pb_data_packet_t *packet = &ev->detail.data_packet; - if (eng->options.on_data_packet) { - eng->options.on_data_packet(packet, eng->options.ctx); - } - break; case EV_SIG_RES: livekit_pb_signal_response_t *res = &ev->detail.res; switch (ev->detail.res.which_message) { From af98bb485158d3d0f921ad4eb6ae9ab19387f759 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:42:33 +1000 Subject: [PATCH 69/81] Add basic connection time benchmarking --- components/livekit/Kconfig | 3 +++ components/livekit/core/peer.c | 18 +++++++++++++++--- components/livekit/core/signaling.c | 11 +++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/components/livekit/Kconfig b/components/livekit/Kconfig index 59dbd21..d8597cf 100644 --- a/components/livekit/Kconfig +++ b/components/livekit/Kconfig @@ -6,6 +6,9 @@ menu "LiveKit" config LK_MAX_ICE_SERVERS int "Maximum number of ICE servers" default 3 + config LK_BENCHMARK + bool "Benchmark connection time" + default n config LK_ENGINE_QUEUE_SIZE int "Number of engine events to queue" default 32 diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index ab99a7d..17027b5 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -6,6 +6,7 @@ #include "esp_webrtc_defaults.h" #include "media_lib_os.h" #include "esp_codec_dev.h" +#include "utils.h" #include "peer.h" @@ -38,6 +39,10 @@ typedef struct { uint16_t reliable_stream_id; uint16_t lossy_stream_id; + +#if CONFIG_LK_BENCHMARK + uint64_t start_time; +#endif } peer_t; static esp_peer_media_dir_t get_media_direction(esp_peer_media_dir_t direction, peer_role_t role) { @@ -67,8 +72,6 @@ static void peer_task(void *ctx) static void create_data_channels(peer_t *peer) { - if (peer->options.role != PEER_ROLE_PUBLISHER) return; - esp_peer_data_channel_cfg_t reliable_cfg = { .label = RELIABLE_CHANNEL_LABEL, .type = ESP_PEER_DATA_CHANNEL_RELIABLE, @@ -106,13 +109,19 @@ static int on_state(esp_peer_state_t rtc_state, void *ctx) new_state = CONNECTION_STATE_CONNECTING; break; case ESP_PEER_STATE_CONNECTED: - create_data_channels(peer); + if (peer->options.role == PEER_ROLE_PUBLISHER) { + create_data_channels(peer); + } break; case ESP_PEER_STATE_DATA_CHANNEL_OPENED: // Don't enter the connected state until both data channels are opened. if (peer->reliable_stream_id == STREAM_ID_INVALID || peer->lossy_stream_id == STREAM_ID_INVALID ) break; new_state = CONNECTION_STATE_CONNECTED; +#if CONFIG_LK_BENCHMARK + ESP_LOGI(TAG(peer), "[BENCH] Connected in %" PRIu64 "ms", + get_unix_time_ms() - peer->start_time); +#endif break; default: break; @@ -327,6 +336,9 @@ peer_err_t peer_connect(peer_handle_t handle) return PEER_ERR_INVALID_ARG; } peer_t *peer = (peer_t *)handle; +#if CONFIG_LK_BENCHMARK + peer->start_time = get_unix_time_ms(); +#endif peer->running = true; media_lib_thread_handle_t thread; diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 7e8137b..69e672c 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -29,6 +29,10 @@ typedef struct { TimerHandle_t ping_interval_timer; TimerHandle_t ping_timeout_timer; int64_t rtt; + +#if CONFIG_LK_BENCHMARK + uint64_t start_time; +#endif } signal_t; static inline void change_state(signal_t *sg, signal_state_t state) @@ -131,6 +135,9 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void switch (event_id) { case WEBSOCKET_EVENT_BEFORE_CONNECT: +#if CONFIG_LK_BENCHMARK + sg->start_time = get_unix_time_ms(); +#endif sg->is_terminal_state = false; change_state(sg, SIGNAL_STATE_CONNECTING); break; @@ -160,6 +167,10 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void change_state(sg, state); break; case WEBSOCKET_EVENT_CONNECTED: +#if CONFIG_LK_BENCHMARK + ESP_LOGI(TAG, "[BENCH] Connected in %" PRIu64 "ms", + get_unix_time_ms() - sg->start_time); +#endif change_state(sg, SIGNAL_STATE_CONNECTED); break; case WEBSOCKET_EVENT_DATA: From bf5a3436fa8b69aa20c108e0bf24e45ed14fcb6b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 12:03:02 +1000 Subject: [PATCH 70/81] Queue peer SDP messages --- components/livekit/core/engine.c | 110 ++++++++++++++++++------------- components/livekit/core/peer.c | 2 +- components/livekit/core/peer.h | 2 +- 3 files changed, 67 insertions(+), 47 deletions(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 5b21a94..07d687b 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -33,6 +33,7 @@ typedef enum { EV_SIG_STATE, /// Signal state changed. EV_SIG_RES, /// Signal response received. EV_PEER_STATE, /// Peer state changed. + EV_PEER_SDP, /// Peer provided SDP. EV_TIMER_EXP, /// Timer expired. EV_MAX_RETRIES_REACHED, /// Maximum number of retry attempts reached. _EV_STATE_ENTER, /// State enter hook (internal). @@ -56,6 +57,12 @@ typedef struct { /// Detail for `EV_SIG_STATE`. signal_state_t sig_state; + /// Detail for `EV_PEER_SDP`. + struct { + const char *sdp; + peer_role_t role; + } peer_sdp; + /// Detail for `EV_PEER_STATE`. struct { connection_state_t state; @@ -145,6 +152,32 @@ static engine_err_t subscribe_tracks(engine_t *eng, livekit_pb_track_info_t *tra return ENGINE_ERR_NONE; } +static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + + av_render_audio_info_t render_info = {}; + convert_dec_aud_info(info, &render_info); + ESP_LOGD(TAG, "Audio render info: codec=%d, sample_rate=%" PRIu32 ", channels=%" PRIu8, + render_info.codec, render_info.sample_rate, render_info.channel); + + if (av_render_add_audio_stream(eng->renderer_handle, &render_info) != ESP_MEDIA_ERR_OK) { + ESP_LOGE(TAG, "Failed to add audio stream to renderer"); + return; + } +} + +static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + av_render_audio_data_t audio_data = { + .pts = frame->pts, + .data = frame->data, + .size = frame->size, + }; + av_render_add_audio_data(eng->renderer_handle, &audio_data); +} + // MARK: - Published media /// Converts `esp_peer_audio_codec_t` to equivalent `esp_capture_codec_type_t` value. @@ -360,6 +393,16 @@ static void on_peer_state_changed(connection_state_t state, peer_role_t role, vo event_enqueue(eng, &ev, true); } +static void on_peer_sdp(const char *sdp, peer_role_t role, void *ctx) +{ + engine_t *eng = (engine_t *)ctx; + engine_event_t ev = { + .type = EV_PEER_SDP, + .detail.peer_sdp = { .sdp = strdup(sdp), .role = role } + }; + event_enqueue(eng, &ev, false); +} + static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) { engine_t *eng = (engine_t *)ctx; @@ -370,48 +413,6 @@ static bool on_peer_data_packet(livekit_pb_data_packet_t* packet, void *ctx) return false; } -// MARK: - Publisher peer event handlers - -static void on_peer_pub_offer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - signal_send_offer(eng->signal_handle, sdp); -} - -// MARK: - Subscriber peer event handlers - -static void on_peer_sub_answer(const char *sdp, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - signal_send_answer(eng->signal_handle, sdp); -} - -static void on_peer_sub_audio_info(esp_peer_audio_stream_info_t* info, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - - av_render_audio_info_t render_info = {}; - convert_dec_aud_info(info, &render_info); - ESP_LOGD(TAG, "Audio render info: codec=%d, sample_rate=%" PRIu32 ", channels=%" PRIu8, - render_info.codec, render_info.sample_rate, render_info.channel); - - if (av_render_add_audio_stream(eng->renderer_handle, &render_info) != ESP_MEDIA_ERR_OK) { - ESP_LOGE(TAG, "Failed to add audio stream to renderer"); - return; - } -} - -static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) -{ - engine_t *eng = (engine_t *)ctx; - av_render_audio_data_t audio_data = { - .pts = frame->pts, - .data = frame->data, - .size = frame->size, - }; - av_render_add_audio_data(eng->renderer_handle, &audio_data); -} - // MARK: - Timer expired handler static void on_timer_expired(TimerHandle_t timer) @@ -511,21 +512,19 @@ static bool establish_peer_connections(engine_t *eng, livekit_pb_join_response_t .server_list = server_list, .server_count = server_count, .on_state_changed = on_peer_state_changed, + .on_sdp = on_peer_sdp, .on_data_packet = on_peer_data_packet, .ctx = eng }; // 1. Publisher options.role = PEER_ROLE_PUBLISHER; - options.on_sdp = on_peer_pub_offer; - _create_and_connect_peer(&options, &eng->pub_peer_handle); if (eng->pub_peer_handle == NULL) return false; // 2. Subscriber options.role = PEER_ROLE_SUBSCRIBER; - options.on_sdp = on_peer_sub_answer; options.on_audio_info = on_peer_sub_audio_info; options.on_audio_frame = on_peer_sub_audio_frame; @@ -619,6 +618,9 @@ static void event_free(engine_event_t *ev) case EV_SIG_RES: protocol_signal_response_free(&ev->detail.res); break; + case EV_PEER_SDP: + SAFE_FREE(ev->detail.peer_sdp.sdp); + break; default: break; } } @@ -851,6 +853,15 @@ static bool handle_state_connecting(engine_t *eng, const engine_event_t *ev) } } break; + case EV_PEER_SDP: + const char *sdp = ev->detail.peer_sdp.sdp; + peer_role_t sdp_role = ev->detail.peer_sdp.role; + if (sdp_role == PEER_ROLE_PUBLISHER) { + signal_send_offer(eng->signal_handle, sdp); + } else { + signal_send_answer(eng->signal_handle, sdp); + } + break; default: break; } @@ -933,6 +944,15 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) eng->state = ENGINE_STATE_BACKOFF; } break; + case EV_PEER_SDP: + const char *sdp = ev->detail.peer_sdp.sdp; + peer_role_t sdp_role = ev->detail.peer_sdp.role; + if (sdp_role != PEER_ROLE_SUBSCRIBER) { + ESP_LOGW(TAG, "Unexpected SDP from publisher"); + break; + } + signal_send_answer(eng->signal_handle, sdp); + break; default: break; } diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index 17027b5..2a0dcda 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -142,7 +142,7 @@ static int on_msg(esp_peer_msg_t *info, void *ctx) ESP_LOGI(TAG(peer), "Generated %s:\n%s", peer->options.role == PEER_ROLE_PUBLISHER ? "offer" : "answer", (char *)info->data); - peer->options.on_sdp((char *)info->data, peer->options.ctx); + peer->options.on_sdp((char *)info->data, peer->options.role, peer->options.ctx); break; default: ESP_LOGD(TAG(peer), "Unhandled msg type: %d", info->type); diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index bebac6b..8f5c0a7 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -56,7 +56,7 @@ typedef struct { /// Invoked when an SDP message is available. This can be either /// an offer or answer depending on target configuration. - void (*on_sdp)(const char *sdp, void *ctx); + void (*on_sdp)(const char *sdp, peer_role_t role, void *ctx); /// Invoked when information about an incoming audio stream is available. void (*on_audio_info)(esp_peer_audio_stream_info_t* info, void *ctx); From 7457ae2b9d89ffd31d9c27ab895c6924fc2a7095 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 12:04:13 +1000 Subject: [PATCH 71/81] Clean up --- components/livekit/core/engine.c | 1 - 1 file changed, 1 deletion(-) diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 07d687b..b969efa 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -902,7 +902,6 @@ static bool handle_state_connected(engine_t *eng, const engine_event_t *ev) livekit_pb_participant_update_t *update = &res->message.update; handle_participant_update(eng, update); break; - // TODO: Only handle if needed case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: livekit_pb_session_description_t *answer = &res->message.answer; peer_handle_sdp(eng->pub_peer_handle, answer->sdp); From bc81a93c8d76bdc1afd2d054a642610c24577b4a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:11:50 +1000 Subject: [PATCH 72/81] Increase timer stack depth --- examples/minimal/sdkconfig.defaults | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/minimal/sdkconfig.defaults b/examples/minimal/sdkconfig.defaults index d9bb373..b40466a 100644 --- a/examples/minimal/sdkconfig.defaults +++ b/examples/minimal/sdkconfig.defaults @@ -15,6 +15,7 @@ CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y CONFIG_ESPTOOLPY_FLASHMODE_QIO=y CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y CONFIG_FREERTOS_HZ=1000 +CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=4096 CONFIG_IDF_EXPERIMENTAL_FEATURES=y CONFIG_LWIP_SNTP_MAX_SERVERS=2 CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y From 7ee5894bfdaffffb5ceb3ff5dffa60e6e3ecb402 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:22:55 +1000 Subject: [PATCH 73/81] Fix typo --- components/livekit/include/livekit.h | 1 + examples/voice_agent/main/example.c | 15 --------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/components/livekit/include/livekit.h b/components/livekit/include/livekit.h index aa3b6f3..5b0d5f8 100644 --- a/components/livekit/include/livekit.h +++ b/components/livekit/include/livekit.h @@ -376,6 +376,7 @@ typedef struct { /// /// Example usage: /// @code +/// const char* command = "G5 I0 J3 P0 Q-3 X2 Y3"; /// livekit_data_payload_t payload = { /// .bytes = (uint8_t*)command, /// .size = strlen(command) diff --git a/examples/voice_agent/main/example.c b/examples/voice_agent/main/example.c index 7d70ac8..3faeb65 100644 --- a/examples/voice_agent/main/example.c +++ b/examples/voice_agent/main/example.c @@ -154,21 +154,6 @@ void join_room() if (connect_res != LIVEKIT_ERR_NONE) { ESP_LOGE(TAG, "Failed to connect to room"); } - - const char* command = "G5 I0 J3 P0 Q-3 X2 Y3"; - - livekit_data_payload_t payload = { - .bytes = (uint8_t*)command, - .size = strlen(command) - }; - livekit_data_publish_options_t options = { - .payload = &payload, - .topic = "gcode", - .lossy = false, - .destination_identities = (char*[]){ "printer-1" }, - .destination_identities_count = 1 - }; - livekit_room_publish_data(room_handle, &options); } void leave_room() From 35de2ca1588d7bfe3c3ffeb189aa97f86a25d728 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:30:20 +1000 Subject: [PATCH 74/81] Document how to get failure reason --- components/livekit/include/livekit.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/components/livekit/include/livekit.h b/components/livekit/include/livekit.h index 5b0d5f8..1c1fd49 100644 --- a/components/livekit/include/livekit.h +++ b/components/livekit/include/livekit.h @@ -252,7 +252,20 @@ livekit_err_t livekit_room_destroy(livekit_room_handle_t handle); /// Connect and disconnect from a room. /// /// The connection state of a room can be monitored by setting a handler for -/// @ref livekit_room_options_t::on_state_changed. +/// @ref livekit_room_options_t::on_state_changed, for example: +/// +/// @code +/// static void on_state_changed(livekit_connection_state_t state, void* ctx) +/// { +/// ESP_LOGI(TAG, "Room state changed: %s", livekit_connection_state_str(state)); +/// +/// // If the connection failed, find out why: +/// livekit_failure_reason_t reason = livekit_room_get_failure_reason(room_handle); +/// if (reason != LIVEKIT_FAILURE_REASON_NONE) { +/// ESP_LOGE(TAG, "Failure reason: %s", livekit_failure_reason_str(reason)); +/// } +/// } +/// @endcode /// /// @{ From 9cf876d65be79063ae4ce729b50ea65665e6158a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:19:01 +1000 Subject: [PATCH 75/81] Data packet optimization - Ignore types that are not supported yet - Fixed length sender SID --- components/livekit/core/peer.c | 5 ++ .../livekit/protocol/livekit_models.pb.h | 78 ++++++++----------- .../protocol/protobufs/livekit_models.options | 6 +- 3 files changed, 41 insertions(+), 48 deletions(-) diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index 2a0dcda..92ec723 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -232,6 +232,11 @@ static int on_data(esp_peer_data_frame_t *frame, void *ctx) ESP_LOGE(TAG(peer), "Failed to decode data packet"); return -1; } + if (packet.which_value == 0) { + // Packet type is not supported yet. + protocol_data_packet_free(&packet); + return -1; + } if (!peer->options.on_data_packet(&packet, peer->options.ctx)) { // Ownership was not taken. protocol_data_packet_free(&packet); diff --git a/components/livekit/protocol/livekit_models.pb.h b/components/livekit/protocol/livekit_models.pb.h index b56cc64..55fb5a2 100644 --- a/components/livekit/protocol/livekit_models.pb.h +++ b/components/livekit/protocol/livekit_models.pb.h @@ -361,6 +361,26 @@ typedef struct livekit_pb_rpc_response { } value; } livekit_pb_rpc_response_t; +/* new DataPacket API */ +typedef struct livekit_pb_data_packet { + pb_size_t which_value; + union { + livekit_pb_user_packet_t user; + livekit_pb_rpc_request_t rpc_request; + livekit_pb_rpc_ack_t rpc_ack; + livekit_pb_rpc_response_t rpc_response; + } value; + /* participant identity of user that sent the message */ + char *participant_identity; + /* identities of participants who will receive the message (sent to all by default) */ + pb_size_t destination_identities_count; + char **destination_identities; + /* sequence number of reliable packet */ + uint32_t sequence; + /* sid of the user that sent the message */ + char participant_sid[16]; +} livekit_pb_data_packet_t; + typedef struct livekit_pb_participant_tracks { /* participant ID of participant to whom the tracks belong */ pb_callback_t participant_sid; @@ -585,30 +605,6 @@ typedef struct livekit_pb_data_stream_trailer { char reason[16]; /* reason why the stream was closed (could contain "error" / "interrupted" / empty for expected end) */ } livekit_pb_data_stream_trailer_t; -/* new DataPacket API */ -typedef struct livekit_pb_data_packet { - pb_size_t which_value; - union { - livekit_pb_user_packet_t user; - livekit_pb_sip_dtmf_t sip_dtmf; - livekit_pb_rpc_request_t rpc_request; - livekit_pb_rpc_ack_t rpc_ack; - livekit_pb_rpc_response_t rpc_response; - livekit_pb_data_stream_header_t stream_header; - livekit_pb_data_stream_chunk_t stream_chunk; - livekit_pb_data_stream_trailer_t stream_trailer; - } value; - /* participant identity of user that sent the message */ - char *participant_identity; - /* identities of participants who will receive the message (sent to all by default) */ - pb_size_t destination_identities_count; - char **destination_identities; - /* sequence number of reliable packet */ - uint32_t sequence; - /* sid of the user that sent the message */ - char *participant_sid; -} livekit_pb_data_packet_t; - typedef struct livekit_pb_webhook_config { pb_callback_t url; pb_callback_t signing_key; @@ -772,7 +768,7 @@ extern "C" { #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_DEFAULT {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} #define LIVEKIT_PB_VIDEO_LAYER_INIT_DEFAULT {_LIVEKIT_PB_VIDEO_QUALITY_MIN, 0, 0} -#define LIVEKIT_PB_DATA_PACKET_INIT_DEFAULT {0, {LIVEKIT_PB_USER_PACKET_INIT_DEFAULT}, NULL, 0, NULL, 0, NULL} +#define LIVEKIT_PB_DATA_PACKET_INIT_DEFAULT {0, {LIVEKIT_PB_USER_PACKET_INIT_DEFAULT}, NULL, 0, NULL, 0, ""} #define LIVEKIT_PB_ACTIVE_SPEAKER_UPDATE_INIT_DEFAULT {{{NULL}, NULL}} #define LIVEKIT_PB_SPEAKER_INFO_INIT_DEFAULT {{{NULL}, NULL}, 0, 0} #define LIVEKIT_PB_USER_PACKET_INIT_DEFAULT {NULL, NULL} @@ -816,7 +812,7 @@ extern "C" { #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_ZERO {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} #define LIVEKIT_PB_VIDEO_LAYER_INIT_ZERO {_LIVEKIT_PB_VIDEO_QUALITY_MIN, 0, 0} -#define LIVEKIT_PB_DATA_PACKET_INIT_ZERO {0, {LIVEKIT_PB_USER_PACKET_INIT_ZERO}, NULL, 0, NULL, 0, NULL} +#define LIVEKIT_PB_DATA_PACKET_INIT_ZERO {0, {LIVEKIT_PB_USER_PACKET_INIT_ZERO}, NULL, 0, NULL, 0, ""} #define LIVEKIT_PB_ACTIVE_SPEAKER_UPDATE_INIT_ZERO {{{NULL}, NULL}} #define LIVEKIT_PB_SPEAKER_INFO_INIT_ZERO {{{NULL}, NULL}, 0, 0} #define LIVEKIT_PB_USER_PACKET_INIT_ZERO {NULL, NULL} @@ -922,6 +918,14 @@ extern "C" { #define LIVEKIT_PB_RPC_RESPONSE_REQUEST_ID_TAG 1 #define LIVEKIT_PB_RPC_RESPONSE_PAYLOAD_TAG 2 #define LIVEKIT_PB_RPC_RESPONSE_ERROR_TAG 3 +#define LIVEKIT_PB_DATA_PACKET_USER_TAG 2 +#define LIVEKIT_PB_DATA_PACKET_RPC_REQUEST_TAG 10 +#define LIVEKIT_PB_DATA_PACKET_RPC_ACK_TAG 11 +#define LIVEKIT_PB_DATA_PACKET_RPC_RESPONSE_TAG 12 +#define LIVEKIT_PB_DATA_PACKET_PARTICIPANT_IDENTITY_TAG 4 +#define LIVEKIT_PB_DATA_PACKET_DESTINATION_IDENTITIES_TAG 5 +#define LIVEKIT_PB_DATA_PACKET_SEQUENCE_TAG 16 +#define LIVEKIT_PB_DATA_PACKET_PARTICIPANT_SID_TAG 17 #define LIVEKIT_PB_PARTICIPANT_TRACKS_PARTICIPANT_SID_TAG 1 #define LIVEKIT_PB_PARTICIPANT_TRACKS_TRACK_SIDS_TAG 2 #define LIVEKIT_PB_SERVER_INFO_EDITION_TAG 1 @@ -1052,18 +1056,6 @@ extern "C" { #define LIVEKIT_PB_DATA_STREAM_CHUNK_VERSION_TAG 4 #define LIVEKIT_PB_DATA_STREAM_TRAILER_STREAM_ID_TAG 1 #define LIVEKIT_PB_DATA_STREAM_TRAILER_REASON_TAG 2 -#define LIVEKIT_PB_DATA_PACKET_USER_TAG 2 -#define LIVEKIT_PB_DATA_PACKET_SIP_DTMF_TAG 6 -#define LIVEKIT_PB_DATA_PACKET_RPC_REQUEST_TAG 10 -#define LIVEKIT_PB_DATA_PACKET_RPC_ACK_TAG 11 -#define LIVEKIT_PB_DATA_PACKET_RPC_RESPONSE_TAG 12 -#define LIVEKIT_PB_DATA_PACKET_STREAM_HEADER_TAG 13 -#define LIVEKIT_PB_DATA_PACKET_STREAM_CHUNK_TAG 14 -#define LIVEKIT_PB_DATA_PACKET_STREAM_TRAILER_TAG 15 -#define LIVEKIT_PB_DATA_PACKET_PARTICIPANT_IDENTITY_TAG 4 -#define LIVEKIT_PB_DATA_PACKET_DESTINATION_IDENTITIES_TAG 5 -#define LIVEKIT_PB_DATA_PACKET_SEQUENCE_TAG 16 -#define LIVEKIT_PB_DATA_PACKET_PARTICIPANT_SID_TAG 17 #define LIVEKIT_PB_WEBHOOK_CONFIG_URL_TAG 1 #define LIVEKIT_PB_WEBHOOK_CONFIG_SIGNING_KEY_TAG 2 @@ -1157,25 +1149,17 @@ X(a, STATIC, SINGULAR, UINT32, height, 3) X(a, STATIC, ONEOF, MESSAGE, (value,user,value.user), 2) \ X(a, POINTER, SINGULAR, STRING, participant_identity, 4) \ X(a, POINTER, REPEATED, STRING, destination_identities, 5) \ -X(a, STATIC, ONEOF, MESSAGE, (value,sip_dtmf,value.sip_dtmf), 6) \ X(a, STATIC, ONEOF, MESSAGE, (value,rpc_request,value.rpc_request), 10) \ X(a, STATIC, ONEOF, MESSAGE, (value,rpc_ack,value.rpc_ack), 11) \ X(a, STATIC, ONEOF, MESSAGE, (value,rpc_response,value.rpc_response), 12) \ -X(a, STATIC, ONEOF, MESSAGE, (value,stream_header,value.stream_header), 13) \ -X(a, STATIC, ONEOF, MESSAGE, (value,stream_chunk,value.stream_chunk), 14) \ -X(a, STATIC, ONEOF, MESSAGE, (value,stream_trailer,value.stream_trailer), 15) \ X(a, STATIC, SINGULAR, UINT32, sequence, 16) \ -X(a, POINTER, SINGULAR, STRING, participant_sid, 17) +X(a, STATIC, SINGULAR, STRING, participant_sid, 17) #define LIVEKIT_PB_DATA_PACKET_CALLBACK NULL #define LIVEKIT_PB_DATA_PACKET_DEFAULT NULL #define livekit_pb_data_packet_t_value_user_MSGTYPE livekit_pb_user_packet_t -#define livekit_pb_data_packet_t_value_sip_dtmf_MSGTYPE livekit_pb_sip_dtmf_t #define livekit_pb_data_packet_t_value_rpc_request_MSGTYPE livekit_pb_rpc_request_t #define livekit_pb_data_packet_t_value_rpc_ack_MSGTYPE livekit_pb_rpc_ack_t #define livekit_pb_data_packet_t_value_rpc_response_MSGTYPE livekit_pb_rpc_response_t -#define livekit_pb_data_packet_t_value_stream_header_MSGTYPE livekit_pb_data_stream_header_t -#define livekit_pb_data_packet_t_value_stream_chunk_MSGTYPE livekit_pb_data_stream_chunk_t -#define livekit_pb_data_packet_t_value_stream_trailer_MSGTYPE livekit_pb_data_stream_trailer_t #define LIVEKIT_PB_ACTIVE_SPEAKER_UPDATE_FIELDLIST(X, a) \ X(a, CALLBACK, REPEATED, MESSAGE, speakers, 1) diff --git a/components/livekit/protocol/protobufs/livekit_models.options b/components/livekit/protocol/protobufs/livekit_models.options index f52dcc2..1086eef 100644 --- a/components/livekit/protocol/protobufs/livekit_models.options +++ b/components/livekit/protocol/protobufs/livekit_models.options @@ -29,10 +29,14 @@ livekit_pb.VideoLayer.ssrc type:FT_IGNORE livekit_pb.DataPacket.metrics type:FT_IGNORE livekit_pb.DataPacket.transcription type:FT_IGNORE +livekit_pb.DataPacket.sip_dtmf type:FT_IGNORE livekit_pb.DataPacket.chat_message type:FT_IGNORE +livekit_pb.DataPacket.stream_header type:FT_IGNORE +livekit_pb.DataPacket.stream_chunk type:FT_IGNORE +livekit_pb.DataPacket.stream_trailer type:FT_IGNORE livekit_pb.DataPacket.participant_identity type:FT_POINTER livekit_pb.DataPacket.destination_identities type:FT_POINTER -livekit_pb.DataPacket.participant_sid type:FT_POINTER +livekit_pb.DataPacket.participant_sid max_length:15 livekit_pb.UserPacket.payload type:FT_POINTER livekit_pb.UserPacket.topic type:FT_POINTER From a5332fa3aaa133fc3ab08e57964ec35debf1a7c9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:37:01 +1000 Subject: [PATCH 76/81] Signal response optimization - Ignore types that are not supported yet --- components/livekit/core/signaling.c | 5 + components/livekit/protocol/livekit_rtc.pb.h | 140 +++++------------- .../protocol/protobufs/livekit_rtc.options | 17 ++- 3 files changed, 56 insertions(+), 106 deletions(-) diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 69e672c..b0f2843 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -182,6 +182,11 @@ static void on_ws_event(void *ctx, esp_event_base_t base, int32_t event_id, void if (!protocol_signal_response_decode((const uint8_t *)data->data_ptr, data->data_len, &res)) { break; } + if (res.which_message == 0) { + // Response type is not supported yet. + protocol_signal_response_free(&res); + break; + } if (!res_middleware(sg, &res)) { // Don't forward. protocol_signal_response_free(&res); diff --git a/components/livekit/protocol/livekit_rtc.pb.h b/components/livekit/protocol/livekit_rtc.pb.h index d80d3bd..ad418bc 100644 --- a/components/livekit/protocol/livekit_rtc.pb.h +++ b/components/livekit/protocol/livekit_rtc.pb.h @@ -354,6 +354,30 @@ typedef struct livekit_pb_pong { int64_t timestamp; } livekit_pb_pong_t; +typedef struct livekit_pb_signal_response { + pb_size_t which_message; + union { + /* sent when join is accepted */ + livekit_pb_join_response_t join; + /* sent when server answers publisher */ + livekit_pb_session_description_t answer; + /* sent when server is sending subscriber an offer */ + livekit_pb_session_description_t offer; + /* sent when an ICE candidate is available */ + livekit_pb_trickle_request_t trickle; + /* sent when participants in the room has changed */ + livekit_pb_participant_update_t update; + /* Immediately terminate session */ + livekit_pb_leave_request_t leave; + /* sent when metadata of the room has changed */ + livekit_pb_room_update_t room_update; + /* respond to ping */ + int64_t pong; /* deprecated by pong_resp (message Pong) */ + /* respond to Ping */ + livekit_pb_pong_t pong_resp; + } message; +} livekit_pb_signal_response_t; + typedef struct livekit_pb_region_settings { pb_callback_t regions; } livekit_pb_region_settings_t; @@ -379,59 +403,6 @@ typedef struct livekit_pb_track_subscribed { char dummy_field; } livekit_pb_track_subscribed_t; -typedef struct livekit_pb_signal_response { - pb_size_t which_message; - union { - /* sent when join is accepted */ - livekit_pb_join_response_t join; - /* sent when server answers publisher */ - livekit_pb_session_description_t answer; - /* sent when server is sending subscriber an offer */ - livekit_pb_session_description_t offer; - /* sent when an ICE candidate is available */ - livekit_pb_trickle_request_t trickle; - /* sent when participants in the room has changed */ - livekit_pb_participant_update_t update; - /* sent to the participant when their track has been published */ - livekit_pb_track_published_response_t track_published; - /* Immediately terminate session */ - livekit_pb_leave_request_t leave; - /* server initiated mute */ - livekit_pb_mute_track_request_t mute; - /* indicates changes to speaker status, including when they've gone to not speaking */ - livekit_pb_speakers_changed_t speakers_changed; - /* sent when metadata of the room has changed */ - livekit_pb_room_update_t room_update; - /* when connection quality changed */ - livekit_pb_connection_quality_update_t connection_quality; - /* when streamed tracks state changed, used to notify when any of the streams were paused due to - congestion */ - livekit_pb_stream_state_update_t stream_state_update; - /* when max subscribe quality changed, used by dynamic broadcasting to disable unused layers */ - livekit_pb_subscribed_quality_update_t subscribed_quality_update; - /* when subscription permission changed */ - livekit_pb_subscription_permission_update_t subscription_permission_update; - /* update the token the client was using, to prevent an active client from using an expired token */ - pb_callback_t refresh_token; - /* server initiated track unpublish */ - livekit_pb_track_unpublished_response_t track_unpublished; - /* respond to ping */ - int64_t pong; /* deprecated by pong_resp (message Pong) */ - /* sent when client reconnects */ - livekit_pb_reconnect_response_t reconnect; - /* respond to Ping */ - livekit_pb_pong_t pong_resp; - /* Subscription response, client should not expect any media from this subscription if it fails */ - livekit_pb_subscription_response_t subscription_response; - /* Response relating to user inititated requests that carry a `request_id` */ - livekit_pb_request_response_t request_response; - /* notify to the publisher when a published track has been subscribed for the first time */ - livekit_pb_track_subscribed_t track_subscribed; - /* notify to the participant when they have been moved to a new room */ - livekit_pb_room_moved_response_t room_moved; - } message; -} livekit_pb_signal_response_t; - #ifdef __cplusplus extern "C" { @@ -735,38 +706,24 @@ extern "C" { #define LIVEKIT_PB_SIGNAL_REQUEST_UPDATE_VIDEO_TRACK_TAG 18 #define LIVEKIT_PB_PONG_LAST_PING_TIMESTAMP_TAG 1 #define LIVEKIT_PB_PONG_TIMESTAMP_TAG 2 -#define LIVEKIT_PB_REGION_SETTINGS_REGIONS_TAG 1 -#define LIVEKIT_PB_REGION_INFO_REGION_TAG 1 -#define LIVEKIT_PB_REGION_INFO_URL_TAG 2 -#define LIVEKIT_PB_REGION_INFO_DISTANCE_TAG 3 -#define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_TRACK_SID_TAG 1 -#define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_ERR_TAG 2 -#define LIVEKIT_PB_REQUEST_RESPONSE_REQUEST_ID_TAG 1 -#define LIVEKIT_PB_REQUEST_RESPONSE_REASON_TAG 2 -#define LIVEKIT_PB_REQUEST_RESPONSE_MESSAGE_TAG 3 #define LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG 1 #define LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG 2 #define LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG 3 #define LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG 4 #define LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG 5 -#define LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_PUBLISHED_TAG 6 #define LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG 8 -#define LIVEKIT_PB_SIGNAL_RESPONSE_MUTE_TAG 9 -#define LIVEKIT_PB_SIGNAL_RESPONSE_SPEAKERS_CHANGED_TAG 10 #define LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG 11 -#define LIVEKIT_PB_SIGNAL_RESPONSE_CONNECTION_QUALITY_TAG 12 -#define LIVEKIT_PB_SIGNAL_RESPONSE_STREAM_STATE_UPDATE_TAG 13 -#define LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIBED_QUALITY_UPDATE_TAG 14 -#define LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIPTION_PERMISSION_UPDATE_TAG 15 -#define LIVEKIT_PB_SIGNAL_RESPONSE_REFRESH_TOKEN_TAG 16 -#define LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_UNPUBLISHED_TAG 17 #define LIVEKIT_PB_SIGNAL_RESPONSE_PONG_TAG 18 -#define LIVEKIT_PB_SIGNAL_RESPONSE_RECONNECT_TAG 19 #define LIVEKIT_PB_SIGNAL_RESPONSE_PONG_RESP_TAG 20 -#define LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIPTION_RESPONSE_TAG 21 -#define LIVEKIT_PB_SIGNAL_RESPONSE_REQUEST_RESPONSE_TAG 22 -#define LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_SUBSCRIBED_TAG 23 -#define LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_MOVED_TAG 24 +#define LIVEKIT_PB_REGION_SETTINGS_REGIONS_TAG 1 +#define LIVEKIT_PB_REGION_INFO_REGION_TAG 1 +#define LIVEKIT_PB_REGION_INFO_URL_TAG 2 +#define LIVEKIT_PB_REGION_INFO_DISTANCE_TAG 3 +#define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_TRACK_SID_TAG 1 +#define LIVEKIT_PB_SUBSCRIPTION_RESPONSE_ERR_TAG 2 +#define LIVEKIT_PB_REQUEST_RESPONSE_REQUEST_ID_TAG 1 +#define LIVEKIT_PB_REQUEST_RESPONSE_REASON_TAG 2 +#define LIVEKIT_PB_REQUEST_RESPONSE_MESSAGE_TAG 3 /* Struct field encoding specification for nanopb */ #define LIVEKIT_PB_SIGNAL_REQUEST_FIELDLIST(X, a) \ @@ -810,47 +767,20 @@ X(a, STATIC, ONEOF, MESSAGE, (message,answer,message.answer), 2) \ X(a, STATIC, ONEOF, MESSAGE, (message,offer,message.offer), 3) \ X(a, STATIC, ONEOF, MESSAGE, (message,trickle,message.trickle), 4) \ X(a, STATIC, ONEOF, MESSAGE, (message,update,message.update), 5) \ -X(a, STATIC, ONEOF, MESSAGE, (message,track_published,message.track_published), 6) \ X(a, STATIC, ONEOF, MESSAGE, (message,leave,message.leave), 8) \ -X(a, STATIC, ONEOF, MESSAGE, (message,mute,message.mute), 9) \ -X(a, STATIC, ONEOF, MESSAGE, (message,speakers_changed,message.speakers_changed), 10) \ X(a, STATIC, ONEOF, MESSAGE, (message,room_update,message.room_update), 11) \ -X(a, STATIC, ONEOF, MESSAGE, (message,connection_quality,message.connection_quality), 12) \ -X(a, STATIC, ONEOF, MESSAGE, (message,stream_state_update,message.stream_state_update), 13) \ -X(a, STATIC, ONEOF, MESSAGE, (message,subscribed_quality_update,message.subscribed_quality_update), 14) \ -X(a, STATIC, ONEOF, MESSAGE, (message,subscription_permission_update,message.subscription_permission_update), 15) \ -X(a, CALLBACK, ONEOF, STRING, (message,refresh_token,message.refresh_token), 16) \ -X(a, STATIC, ONEOF, MESSAGE, (message,track_unpublished,message.track_unpublished), 17) \ X(a, STATIC, ONEOF, INT64, (message,pong,message.pong), 18) \ -X(a, STATIC, ONEOF, MESSAGE, (message,reconnect,message.reconnect), 19) \ -X(a, STATIC, ONEOF, MESSAGE, (message,pong_resp,message.pong_resp), 20) \ -X(a, STATIC, ONEOF, MESSAGE, (message,subscription_response,message.subscription_response), 21) \ -X(a, STATIC, ONEOF, MESSAGE, (message,request_response,message.request_response), 22) \ -X(a, STATIC, ONEOF, MESSAGE, (message,track_subscribed,message.track_subscribed), 23) \ -X(a, STATIC, ONEOF, MESSAGE, (message,room_moved,message.room_moved), 24) -#define LIVEKIT_PB_SIGNAL_RESPONSE_CALLBACK pb_default_field_callback +X(a, STATIC, ONEOF, MESSAGE, (message,pong_resp,message.pong_resp), 20) +#define LIVEKIT_PB_SIGNAL_RESPONSE_CALLBACK NULL #define LIVEKIT_PB_SIGNAL_RESPONSE_DEFAULT NULL #define livekit_pb_signal_response_t_message_join_MSGTYPE livekit_pb_join_response_t #define livekit_pb_signal_response_t_message_answer_MSGTYPE livekit_pb_session_description_t #define livekit_pb_signal_response_t_message_offer_MSGTYPE livekit_pb_session_description_t #define livekit_pb_signal_response_t_message_trickle_MSGTYPE livekit_pb_trickle_request_t #define livekit_pb_signal_response_t_message_update_MSGTYPE livekit_pb_participant_update_t -#define livekit_pb_signal_response_t_message_track_published_MSGTYPE livekit_pb_track_published_response_t #define livekit_pb_signal_response_t_message_leave_MSGTYPE livekit_pb_leave_request_t -#define livekit_pb_signal_response_t_message_mute_MSGTYPE livekit_pb_mute_track_request_t -#define livekit_pb_signal_response_t_message_speakers_changed_MSGTYPE livekit_pb_speakers_changed_t #define livekit_pb_signal_response_t_message_room_update_MSGTYPE livekit_pb_room_update_t -#define livekit_pb_signal_response_t_message_connection_quality_MSGTYPE livekit_pb_connection_quality_update_t -#define livekit_pb_signal_response_t_message_stream_state_update_MSGTYPE livekit_pb_stream_state_update_t -#define livekit_pb_signal_response_t_message_subscribed_quality_update_MSGTYPE livekit_pb_subscribed_quality_update_t -#define livekit_pb_signal_response_t_message_subscription_permission_update_MSGTYPE livekit_pb_subscription_permission_update_t -#define livekit_pb_signal_response_t_message_track_unpublished_MSGTYPE livekit_pb_track_unpublished_response_t -#define livekit_pb_signal_response_t_message_reconnect_MSGTYPE livekit_pb_reconnect_response_t #define livekit_pb_signal_response_t_message_pong_resp_MSGTYPE livekit_pb_pong_t -#define livekit_pb_signal_response_t_message_subscription_response_MSGTYPE livekit_pb_subscription_response_t -#define livekit_pb_signal_response_t_message_request_response_MSGTYPE livekit_pb_request_response_t -#define livekit_pb_signal_response_t_message_track_subscribed_MSGTYPE livekit_pb_track_subscribed_t -#define livekit_pb_signal_response_t_message_room_moved_MSGTYPE livekit_pb_room_moved_response_t #define LIVEKIT_PB_SIMULCAST_CODEC_FIELDLIST(X, a) \ X(a, CALLBACK, SINGULAR, STRING, codec, 1) \ diff --git a/components/livekit/protocol/protobufs/livekit_rtc.options b/components/livekit/protocol/protobufs/livekit_rtc.options index 749a32c..147ea9d 100644 --- a/components/livekit/protocol/protobufs/livekit_rtc.options +++ b/components/livekit/protocol/protobufs/livekit_rtc.options @@ -43,4 +43,19 @@ livekit_pb.TrackSubscribed.track_sid type:FT_IGNORE livekit_pb.ParticipantUpdate.participants type:FT_POINTER livekit_pb.UpdateSubscription.track_sids type:FT_POINTER -livekit_pb.UpdateSubscription.participant_tracks type:FT_IGNORE \ No newline at end of file +livekit_pb.UpdateSubscription.participant_tracks type:FT_IGNORE + +livekit_pb.SignalResponse.connection_quality type:FT_IGNORE +livekit_pb.SignalResponse.subscription_permission_update type:FT_IGNORE +livekit_pb.SignalResponse.track_published type:FT_IGNORE +livekit_pb.SignalResponse.track_subscribed type:FT_IGNORE +livekit_pb.SignalResponse.mute type:FT_IGNORE +livekit_pb.SignalResponse.speakers_changed type:FT_IGNORE +livekit_pb.SignalResponse.stream_state_update type:FT_IGNORE +livekit_pb.SignalResponse.subscribed_quality_update type:FT_IGNORE +livekit_pb.SignalResponse.refresh_token type:FT_IGNORE +livekit_pb.SignalResponse.track_unpublished type:FT_IGNORE +livekit_pb.SignalResponse.reconnect type:FT_IGNORE +livekit_pb.SignalResponse.subscription_response type:FT_IGNORE +livekit_pb.SignalResponse.request_response type:FT_IGNORE +livekit_pb.SignalResponse.room_moved type:FT_IGNORE \ No newline at end of file From 5d23758ef6ba5b3d14197147e990e4d86f4ca9e9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:58:14 +1000 Subject: [PATCH 77/81] Fix incorrect field type --- components/livekit/protocol/livekit_models.pb.h | 8 ++++---- .../livekit/protocol/protobufs/livekit_models.options | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/components/livekit/protocol/livekit_models.pb.h b/components/livekit/protocol/livekit_models.pb.h index 55fb5a2..debd389 100644 --- a/components/livekit/protocol/livekit_models.pb.h +++ b/components/livekit/protocol/livekit_models.pb.h @@ -252,7 +252,7 @@ typedef struct livekit_pb_participant_info { char *metadata; char *name; livekit_pb_participant_permission_t permission; - livekit_pb_participant_info_kind_t *kind; + livekit_pb_participant_info_kind_t kind; } livekit_pb_participant_info_t; typedef struct livekit_pb_encryption { @@ -763,7 +763,7 @@ extern "C" { #define LIVEKIT_PB_CODEC_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_PLAYOUT_DELAY_INIT_DEFAULT {0, 0, 0} #define LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT {0, 0, 0} -#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_DEFAULT {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT, NULL} +#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_DEFAULT {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_DEFAULT, _LIVEKIT_PB_PARTICIPANT_INFO_KIND_MIN} #define LIVEKIT_PB_ENCRYPTION_INIT_DEFAULT {0} #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_DEFAULT {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_DEFAULT {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} @@ -807,7 +807,7 @@ extern "C" { #define LIVEKIT_PB_CODEC_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_PLAYOUT_DELAY_INIT_ZERO {0, 0, 0} #define LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO {0, 0, 0} -#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_ZERO {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO, NULL} +#define LIVEKIT_PB_PARTICIPANT_INFO_INIT_ZERO {"", NULL, _LIVEKIT_PB_PARTICIPANT_INFO_STATE_MIN, 0, NULL, NULL, NULL, LIVEKIT_PB_PARTICIPANT_PERMISSION_INIT_ZERO, _LIVEKIT_PB_PARTICIPANT_INFO_KIND_MIN} #define LIVEKIT_PB_ENCRYPTION_INIT_ZERO {0} #define LIVEKIT_PB_SIMULCAST_CODEC_INFO_INIT_ZERO {{{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}, {{NULL}, NULL}} #define LIVEKIT_PB_TRACK_INFO_INIT_ZERO {NULL, _LIVEKIT_PB_TRACK_TYPE_MIN, 0, NULL, 0, 0, {_LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN, _LIVEKIT_PB_AUDIO_TRACK_FEATURE_MIN}} @@ -1108,7 +1108,7 @@ X(a, POINTER, REPEATED, MESSAGE, tracks, 4) \ X(a, POINTER, SINGULAR, STRING, metadata, 5) \ X(a, POINTER, SINGULAR, STRING, name, 9) \ X(a, STATIC, REQUIRED, MESSAGE, permission, 11) \ -X(a, POINTER, SINGULAR, UENUM, kind, 14) +X(a, STATIC, SINGULAR, UENUM, kind, 14) #define LIVEKIT_PB_PARTICIPANT_INFO_CALLBACK NULL #define LIVEKIT_PB_PARTICIPANT_INFO_DEFAULT NULL #define livekit_pb_participant_info_t_tracks_MSGTYPE livekit_pb_track_info_t diff --git a/components/livekit/protocol/protobufs/livekit_models.options b/components/livekit/protocol/protobufs/livekit_models.options index 1086eef..3c64749 100644 --- a/components/livekit/protocol/protobufs/livekit_models.options +++ b/components/livekit/protocol/protobufs/livekit_models.options @@ -13,7 +13,6 @@ livekit_pb.ParticipantInfo.version type:FT_IGNORE livekit_pb.ParticipantInfo.permission label_override:LABEL_REQUIRED livekit_pb.ParticipantInfo.region type:FT_IGNORE livekit_pb.ParticipantInfo.is_publisher type:FT_IGNORE -livekit_pb.ParticipantInfo.kind type:FT_POINTER livekit_pb.ParticipantInfo.attributes type:FT_IGNORE livekit_pb.ParticipantInfo.disconnect_reason type:FT_IGNORE livekit_pb.ParticipantInfo.kind_details type:FT_IGNORE From 69e63afbff76c4bae6f6d2a16c6c23f12766cd06 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:05:10 +1000 Subject: [PATCH 78/81] Fix voice agent example agent join --- examples/voice_agent/main/example.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/voice_agent/main/example.c b/examples/voice_agent/main/example.c index 3faeb65..ba4b5ee 100644 --- a/examples/voice_agent/main/example.c +++ b/examples/voice_agent/main/example.c @@ -10,6 +10,7 @@ static const char *TAG = "livekit_example"; static livekit_room_handle_t room_handle; +static bool agent_joined = false; /// Invoked when the room's connection state changes. static void on_state_changed(livekit_connection_state_t state, void* ctx) @@ -24,18 +25,16 @@ static void on_participant_info(const livekit_participant_info_t* info, void* ct // Only handle agent participants for this example. return; } - char* verb; + bool joined = false; switch (info->state) { - case LIVEKIT_PARTICIPANT_STATE_ACTIVE: - verb = "joined"; - break; - case LIVEKIT_PARTICIPANT_STATE_DISCONNECTED: - verb = "left"; - break; - default: - return; + case LIVEKIT_PARTICIPANT_STATE_ACTIVE: joined = true; break; + case LIVEKIT_PARTICIPANT_STATE_DISCONNECTED: joined = false; break; + default: return; + } + if (joined != agent_joined) { + ESP_LOGI(TAG, "Agent has %s the room", joined ? "joined" : "left"); + agent_joined = joined; } - ESP_LOGI(TAG, "Agent has %s the room", verb); } /// Invoked by a remote participant to set the state of an on-board LED. From 62d7a2c09025429afd9a97d1d5532f678c0968ba Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:07:14 +1000 Subject: [PATCH 79/81] Demonstrate getting failure reason in all examples --- examples/minimal/main/example.c | 1 + examples/voice_agent/main/example.c | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/minimal/main/example.c b/examples/minimal/main/example.c index f7a8f32..729792b 100644 --- a/examples/minimal/main/example.c +++ b/examples/minimal/main/example.c @@ -13,6 +13,7 @@ static livekit_room_handle_t room_handle; static void on_state_changed(livekit_connection_state_t state, void* ctx) { ESP_LOGI(TAG, "Room state changed: %s", livekit_connection_state_str(state)); + livekit_failure_reason_t reason = livekit_room_get_failure_reason(room_handle); if (reason != LIVEKIT_FAILURE_REASON_NONE) { ESP_LOGE(TAG, "Failure reason: %s", livekit_failure_reason_str(reason)); diff --git a/examples/voice_agent/main/example.c b/examples/voice_agent/main/example.c index ba4b5ee..c230ccc 100644 --- a/examples/voice_agent/main/example.c +++ b/examples/voice_agent/main/example.c @@ -15,7 +15,12 @@ static bool agent_joined = false; /// Invoked when the room's connection state changes. static void on_state_changed(livekit_connection_state_t state, void* ctx) { - ESP_LOGI(TAG, "Room state: %s", livekit_connection_state_str(state)); + ESP_LOGI(TAG, "Room state changed: %s", livekit_connection_state_str(state)); + + livekit_failure_reason_t reason = livekit_room_get_failure_reason(room_handle); + if (reason != LIVEKIT_FAILURE_REASON_NONE) { + ESP_LOGE(TAG, "Failure reason: %s", livekit_failure_reason_str(reason)); + } } /// Invoked when participant information is received. From aa8abd69744efd88244e9c0e49f75518d5f95c95 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:08:24 +1000 Subject: [PATCH 80/81] Consistent sdkconfig defaults between examples --- examples/voice_agent/sdkconfig.defaults | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/voice_agent/sdkconfig.defaults b/examples/voice_agent/sdkconfig.defaults index b10f30d..ce22086 100644 --- a/examples/voice_agent/sdkconfig.defaults +++ b/examples/voice_agent/sdkconfig.defaults @@ -36,4 +36,6 @@ CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y CONFIG_IDF_TARGET_ESP32P4=1 # Board support package -CONFIG_BSP_I2C_NUM=0 \ No newline at end of file +CONFIG_BSP_I2C_NUM=0 + +CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=4096 \ No newline at end of file From e060f64cdbd3b7fbd82ba282d87573439bd7a47d Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:42:48 +1000 Subject: [PATCH 81/81] Bump version --- components/livekit/idf_component.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/livekit/idf_component.yml b/components/livekit/idf_component.yml index 9a336e1..6b107d0 100644 --- a/components/livekit/idf_component.yml +++ b/components/livekit/idf_component.yml @@ -4,7 +4,7 @@ url: "https://livekit.io" repository: "https://github.com/livekit/client-sdk-esp32" issues: "https://github.com/livekit/client-sdk-esp32/issues" discussion: "https://livekit.io/join-slack" -version: 0.1.0 +version: 0.2.0 dependencies: idf: ">=5.4" espressif/esp_websocket_client: ^1.4.0