From d2934eca7e62338a586bb617e406b16e81049019 Mon Sep 17 00:00:00 2001 From: jp9000 Date: Sun, 5 Feb 2017 21:37:35 -0800 Subject: [PATCH] libobs: Implement audio monitoring Adds functions to turn on audio monitoring to allow the user to hear playback of an audio source over the user's speaker. It can be set to turn off monitoring and only output to stream, or it can be set to output only to monitoring, or it can be set to both. On windows, audio monitoring uses WASAPI. Windows also is capable of syncing the audio to the video according to when the video frame itself was played. On mac, it uses AudioQueue. On linux, it's not currently implemented and won't do anything (to be implemented). --- libobs/CMakeLists.txt | 39 +- .../null/null-audio-monitoring.c | 23 + .../osx/coreaudio-enum-devices.c | 96 +++++ .../audio-monitoring/osx/coreaudio-output.c | 322 ++++++++++++++ libobs/audio-monitoring/osx/mac-helpers.h | 15 + .../win32/wasapi-enum-devices.c | 105 +++++ libobs/audio-monitoring/win32/wasapi-output.c | 402 ++++++++++++++++++ libobs/audio-monitoring/win32/wasapi-output.h | 13 + libobs/obs-internal.h | 16 +- libobs/obs-source.c | 48 ++- libobs/obs-source.h | 8 + libobs/obs.c | 70 ++- libobs/obs.h | 20 + 13 files changed, 1170 insertions(+), 7 deletions(-) create mode 100644 libobs/audio-monitoring/null/null-audio-monitoring.c create mode 100644 libobs/audio-monitoring/osx/coreaudio-enum-devices.c create mode 100644 libobs/audio-monitoring/osx/coreaudio-output.c create mode 100644 libobs/audio-monitoring/osx/mac-helpers.h create mode 100644 libobs/audio-monitoring/win32/wasapi-enum-devices.c create mode 100644 libobs/audio-monitoring/win32/wasapi-output.c create mode 100644 libobs/audio-monitoring/win32/wasapi-output.h diff --git a/libobs/CMakeLists.txt b/libobs/CMakeLists.txt index a9f4ac9ede07d..cd2b80e1ef2cb 100644 --- a/libobs/CMakeLists.txt +++ b/libobs/CMakeLists.txt @@ -68,6 +68,13 @@ if(WIN32) util/windows/CoTaskMemPtr.hpp util/windows/HRError.hpp util/windows/WinHandle.hpp) + set(libobs_audio_monitoring_SOURCES + audio-monitoring/win32/wasapi-output.c + audio-monitoring/win32/wasapi-enum-devices.c + ) + set(libobs_audio_monitoring_HEADERS + audio-monitoring/win32/wasapi-output.h + ) set(libobs_PLATFORM_DEPS winmm) if(MSVC) set(libobs_PLATFORM_DEPS @@ -83,6 +90,13 @@ elseif(APPLE) util/platform-cocoa.m) set(libobs_PLATFORM_HEADERS util/threading-posix.h) + set(libobs_audio_monitoring_SOURCES + audio-monitoring/osx/coreaudio-enum-devices.c + audio-monitoring/osx/coreaudio-output.c + ) + set(libobs_audio_monitoring_HEADERS + audio-monitoring/osx/mac-helpers.h + ) set_source_files_properties(${libobs_PLATFORM_SOURCES} PROPERTIES @@ -93,6 +107,18 @@ elseif(APPLE) mark_as_advanced(COCOA) include_directories(${COCOA}) + find_library(COREAUDIO CoreAudio) + mark_as_advanced(COREAUDIO) + include_directories(${COREAUDIO}) + + find_library(AUDIOTOOLBOX AudioToolbox) + mark_as_advanced(AUDIOTOOLBOX) + include_directories(${AUDIOTOOLBOX}) + + find_library(AUDIOUNIT AudioUnit) + mark_as_advanced(AUDIOUNIT) + include_directories(${AUDIOUNIT}) + find_library(APPKIT AppKit) mark_as_advanced(APPKIT) include_directories(${APPKIT}) @@ -107,6 +133,9 @@ elseif(APPLE) set(libobs_PLATFORM_DEPS ${COCOA} + ${COREAUDIO} + ${AUDIOUNIT} + ${AUDIOTOOLBOX} ${APPKIT} ${IOKIT} ${CARBON}) @@ -118,6 +147,9 @@ elseif(UNIX) util/platform-nix.c) set(libobs_PLATFORM_HEADERS util/threading-posix.h) + set(libobs_audio_monitoring_SOURCES + audio-monitoring/null/null-audio-monitoring.c + ) if(DBUS_FOUND) set(libobs_PLATFORM_SOURCES ${libobs_PLATFORM_SOURCES} @@ -334,7 +366,10 @@ set(libobs_HEADERS ${libobs_graphics_HEADERS} ${libobs_mediaio_HEADERS} ${libobs_util_HEADERS} - ${libobs_libobs_HEADERS}) + ${libobs_libobs_HEADERS} + ${libobs_audio_monitoring_SOURCES} + ${libobs_audio_monitoring_HEADERS} + ) source_group("callback\\Source Files" FILES ${libobs_callback_SOURCES}) source_group("callback\\Header Files" FILES ${libobs_callback_HEADERS}) @@ -346,6 +381,8 @@ source_group("media-io\\Source Files" FILES ${libobs_mediaio_SOURCES}) source_group("media-io\\Header Files" FILES ${libobs_mediaio_HEADERS}) source_group("util\\Source Files" FILES ${libobs_util_SOURCES}) source_group("util\\Header Files" FILES ${libobs_util_HEADERS}) +source_group("audio-monitoring\\Source Files" FILES ${libobs_audio_monitoring_SOURCES}) +source_group("audio-monitoring\\Header Files" FILES ${libobs_audio_monitoring_HEADERS}) if(BUILD_CAPTIONS) include_directories(${CMAKE_SOURCE_DIR}/deps/libcaption) diff --git a/libobs/audio-monitoring/null/null-audio-monitoring.c b/libobs/audio-monitoring/null/null-audio-monitoring.c new file mode 100644 index 0000000000000..9d7b2a4edaa22 --- /dev/null +++ b/libobs/audio-monitoring/null/null-audio-monitoring.c @@ -0,0 +1,23 @@ +#include "../../obs-internal.h" + +void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, void *data) +{ + UNUSED_PARAMETER(cb); + UNUSED_PARAMETER(data); +} + +struct audio_monitor *audio_monitor_create(obs_source_t *source) +{ + UNUSED_PARAMETER(source); + return NULL; +} + +void audio_monitor_reset(struct audio_monitor *monitor) +{ + UNUSED_PARAMETER(monitor); +} + +void audio_monitor_destroy(struct audio_monitor *monitor) +{ + UNUSED_PARAMETER(monitor); +} diff --git a/libobs/audio-monitoring/osx/coreaudio-enum-devices.c b/libobs/audio-monitoring/osx/coreaudio-enum-devices.c new file mode 100644 index 0000000000000..5fd591bb0b0d1 --- /dev/null +++ b/libobs/audio-monitoring/osx/coreaudio-enum-devices.c @@ -0,0 +1,96 @@ +#include +#include + +#include "../../obs-internal.h" +#include "../../util/dstr.h" + +#include "mac-helpers.h" + +static inline bool cf_to_cstr(CFStringRef ref, char *buf, size_t size) +{ + if (!ref) return false; + return (bool)CFStringGetCString(ref, buf, size, kCFStringEncodingUTF8); +} + +static void obs_enum_audio_monitoring_device(obs_enum_audio_device_cb cb, + void *data, AudioDeviceID id) +{ + UInt32 size = 0; + CFStringRef cf_name = NULL; + CFStringRef cf_uid = NULL; + char name[1024]; + char uid[1024]; + OSStatus stat; + + AudioObjectPropertyAddress addr = { + kAudioDevicePropertyStreams, + kAudioDevicePropertyScopeInput, + kAudioObjectPropertyElementMaster + }; + + /* check to see if it's a mac input device */ + AudioObjectGetPropertyDataSize(id, &addr, 0, NULL, &size); + if (!size) + return; + + size = sizeof(CFStringRef); + + addr.mSelector = kAudioDevicePropertyDeviceUID; + stat = AudioObjectGetPropertyData(id, &addr, 0, NULL, &size, &cf_uid); + if (!success(stat, "get audio device UID")) + return; + + addr.mSelector = kAudioDevicePropertyDeviceNameCFString; + stat = AudioObjectGetPropertyData(id, &addr, 0, NULL, &size, &cf_name); + if (!success(stat, "get audio device name")) + goto fail; + + if (!cf_to_cstr(cf_name, name, sizeof(name))) { + blog(LOG_WARNING, "%s: failed to convert name", __FUNCTION__); + goto fail; + } + + if (!cf_to_cstr(cf_uid, uid, sizeof(uid))) { + blog(LOG_WARNING, "%s: failed to convert uid", __FUNCTION__); + goto fail; + } + + cb(data, name, uid); + +fail: + if (cf_name) + CFRelease(cf_name); + if (cf_uid) + CFRelease(cf_uid); +} + +void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, void *data) +{ + AudioObjectPropertyAddress addr = { + kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster + }; + + UInt32 size = 0; + UInt32 count; + OSStatus stat; + AudioDeviceID *ids; + + stat = AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &addr, + 0, NULL, &size); + if (!success(stat, "get data size")) + return; + + ids = malloc(size); + count = size / sizeof(AudioDeviceID); + + stat = AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, + 0, NULL, &size, ids); + if (success(stat, "get data")) { + for (UInt32 i = 0; i < count; i++) + obs_enum_audio_monitoring_device(cb, data, ids[i]); + } + + free(ids); +} diff --git a/libobs/audio-monitoring/osx/coreaudio-output.c b/libobs/audio-monitoring/osx/coreaudio-output.c new file mode 100644 index 0000000000000..52ff672841cd9 --- /dev/null +++ b/libobs/audio-monitoring/osx/coreaudio-output.c @@ -0,0 +1,322 @@ +#include +#include +#include +#include + +#include "../../media-io/audio-resampler.h" +#include "../../util/circlebuf.h" +#include "../../util/threading.h" +#include "../../util/platform.h" +#include "../../obs-internal.h" +#include "../../util/darray.h" + +#include "mac-helpers.h" + +struct audio_monitor { + obs_source_t *source; + AudioQueueRef queue; + AudioQueueBufferRef buffers[3]; + + pthread_mutex_t mutex; + struct circlebuf empty_buffers; + struct circlebuf new_data; + audio_resampler_t *resampler; + size_t buffer_size; + size_t wait_size; + uint32_t channels; + + volatile bool active; + bool paused; +}; + +static inline bool fill_buffer(struct audio_monitor *monitor) +{ + AudioQueueBufferRef buf; + OSStatus stat; + + if (monitor->new_data.size < monitor->buffer_size) { + return false; + } + + circlebuf_pop_front(&monitor->empty_buffers, &buf, sizeof(buf)); + circlebuf_pop_front(&monitor->new_data, buf->mAudioData, + monitor->buffer_size); + + buf->mAudioDataByteSize = monitor->buffer_size; + + stat = AudioQueueEnqueueBuffer(monitor->queue, buf, 0, NULL); + if (!success(stat, "AudioQueueEnqueueBuffer")) { + blog(LOG_WARNING, "%s: %s", __FUNCTION__, + "Failed to enqueue buffer"); + AudioQueueStop(monitor->queue, false); + } + return true; +} + +static void on_audio_playback(void *param, obs_source_t *source, + const struct audio_data *audio_data, bool muted) +{ + struct audio_monitor *monitor = param; + float vol = source->user_volume; + uint32_t bytes; + + UNUSED_PARAMETER(source); + + if (!os_atomic_load_bool(&monitor->active)) { + return; + } + + uint8_t *resample_data[MAX_AV_PLANES]; + uint32_t resample_frames; + uint64_t ts_offset; + bool success; + + success = audio_resampler_resample(monitor->resampler, resample_data, + &resample_frames, &ts_offset, + (const uint8_t *const *)audio_data->data, + (uint32_t)audio_data->frames); + if (!success) { + return; + } + + bytes = sizeof(float) * monitor->channels * resample_frames; + + if (muted) { + memset(resample_data[0], 0, bytes); + } else { + /* apply volume */ + if (!close_float(vol, 1.0f, EPSILON)) { + register float *cur = (float*)resample_data[0]; + register float *end = cur + + resample_frames * monitor->channels; + + while (cur < end) + *(cur++) *= vol; + } + } + + pthread_mutex_lock(&monitor->mutex); + circlebuf_push_back(&monitor->new_data, resample_data[0], bytes); + + if (monitor->new_data.size >= monitor->wait_size) { + monitor->wait_size = 0; + + while (monitor->empty_buffers.size > 0) { + if (!fill_buffer(monitor)) { + break; + } + } + + if (monitor->paused) { + AudioQueueStart(monitor->queue, NULL); + monitor->paused = false; + } + } + + pthread_mutex_unlock(&monitor->mutex); +} + +static void buffer_audio(void *data, AudioQueueRef aq, AudioQueueBufferRef buf) +{ + struct audio_monitor *monitor = data; + + pthread_mutex_lock(&monitor->mutex); + circlebuf_push_back(&monitor->empty_buffers, &buf, sizeof(buf)); + while (monitor->empty_buffers.size > 0) { + if (!fill_buffer(monitor)) { + break; + } + } + if (monitor->empty_buffers.size == sizeof(buf) * 3) { + monitor->paused = true; + monitor->wait_size = monitor->buffer_size * 3; + AudioQueuePause(monitor->queue); + } + pthread_mutex_unlock(&monitor->mutex); + + UNUSED_PARAMETER(aq); +} + +static bool audio_monitor_init(struct audio_monitor *monitor) +{ + const struct audio_output_info *info = audio_output_get_info( + obs->audio.audio); + uint32_t channels = get_audio_channels(info->speakers); + OSStatus stat; + + AudioStreamBasicDescription desc = { + .mSampleRate = (Float64)info->samples_per_sec, + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kAudioFormatFlagIsFloat | + kAudioFormatFlagIsPacked, + .mBytesPerPacket = sizeof(float) * channels, + .mFramesPerPacket = 1, + .mBytesPerFrame = sizeof(float) * channels, + .mChannelsPerFrame = channels, + .mBitsPerChannel = sizeof(float) * 8 + }; + + monitor->channels = channels; + monitor->buffer_size = + channels * sizeof(float) * info->samples_per_sec / 100 * 3; + monitor->wait_size = monitor->buffer_size * 3; + + pthread_mutex_init_value(&monitor->mutex); + + stat = AudioQueueNewOutput(&desc, buffer_audio, monitor, NULL, NULL, 0, + &monitor->queue); + if (!success(stat, "AudioStreamBasicDescription")) { + return false; + } + + const char *uid = obs->audio.monitoring_device_id; + if (!uid || !*uid) { + return false; + } + + if (strcmp(uid, "default") != 0) { + CFStringRef cf_uid = CFStringCreateWithBytesNoCopy(NULL, + (const UInt8*)uid, strlen(uid), + kCFStringEncodingUTF8, + false, NULL); + + stat = AudioQueueSetProperty(monitor->queue, + kAudioQueueProperty_CurrentDevice, + cf_uid, sizeof(cf_uid)); + CFRelease(cf_uid); + + if (!success(stat, "set current device")) { + return false; + } + } + + stat = AudioQueueSetParameter(monitor->queue, + kAudioQueueParam_Volume, 1.0); + if (!success(stat, "set volume")) { + return false; + } + + for (size_t i = 0; i < 3; i++) { + stat = AudioQueueAllocateBuffer(monitor->queue, + monitor->buffer_size, &monitor->buffers[i]); + if (!success(stat, "allocation of buffer")) { + return false; + } + + circlebuf_push_back(&monitor->empty_buffers, + &monitor->buffers[i], + sizeof(monitor->buffers[i])); + } + + if (pthread_mutex_init(&monitor->mutex, NULL) != 0) { + blog(LOG_WARNING, "%s: %s", __FUNCTION__, + "Failed to init mutex"); + return false; + } + + struct resample_info from = { + .samples_per_sec = info->samples_per_sec, + .speakers = info->speakers, + .format = AUDIO_FORMAT_FLOAT_PLANAR + }; + struct resample_info to = { + .samples_per_sec = info->samples_per_sec, + .speakers = info->speakers, + .format = AUDIO_FORMAT_FLOAT + }; + + monitor->resampler = audio_resampler_create(&to, &from); + if (!monitor->resampler) { + blog(LOG_WARNING, "%s: %s", __FUNCTION__, + "Failed to create resampler"); + return false; + } + + stat = AudioQueueStart(monitor->queue, NULL); + if (!success(stat, "start")) { + return false; + } + + monitor->active = true; + return true; +} + +static void audio_monitor_free(struct audio_monitor *monitor) +{ + if (monitor->source) { + obs_source_remove_audio_capture_callback( + monitor->source, on_audio_playback, monitor); + } + if (monitor->active) { + AudioQueueStop(monitor->queue, true); + } + for (size_t i = 0; i < 3; i++) { + if (monitor->buffers[i]) { + AudioQueueFreeBuffer(monitor->queue, + monitor->buffers[i]); + } + } + if (monitor->queue) { + AudioQueueDispose(monitor->queue, true); + } + + audio_resampler_destroy(monitor->resampler); + circlebuf_free(&monitor->empty_buffers); + circlebuf_free(&monitor->new_data); + pthread_mutex_destroy(&monitor->mutex); +} + +static void audio_monitor_init_final(struct audio_monitor *monitor, + obs_source_t *source) +{ + monitor->source = source; + obs_source_add_audio_capture_callback(source, on_audio_playback, + monitor); +} + +struct audio_monitor *audio_monitor_create(obs_source_t *source) +{ + struct audio_monitor *monitor = bzalloc(sizeof(*monitor)); + + if (!audio_monitor_init(monitor)) { + goto fail; + } + + pthread_mutex_lock(&obs->audio.monitoring_mutex); + da_push_back(obs->audio.monitors, &monitor); + pthread_mutex_unlock(&obs->audio.monitoring_mutex); + + audio_monitor_init_final(monitor, source); + return monitor; + +fail: + audio_monitor_free(monitor); + bfree(monitor); + return NULL; +} + +void audio_monitor_reset(struct audio_monitor *monitor) +{ + bool success; + + obs_source_t *source = monitor->source; + audio_monitor_free(monitor); + memset(monitor, 0, sizeof(*monitor)); + + success = audio_monitor_init(monitor); + if (success) + audio_monitor_init_final(monitor, source); +} + +void audio_monitor_destroy(struct audio_monitor *monitor) +{ + if (monitor) { + audio_monitor_free(monitor); + + pthread_mutex_lock(&obs->audio.monitoring_mutex); + da_erase_item(obs->audio.monitors, &monitor); + pthread_mutex_unlock(&obs->audio.monitoring_mutex); + + bfree(monitor); + } +} diff --git a/libobs/audio-monitoring/osx/mac-helpers.h b/libobs/audio-monitoring/osx/mac-helpers.h new file mode 100644 index 0000000000000..f995990a0bf3c --- /dev/null +++ b/libobs/audio-monitoring/osx/mac-helpers.h @@ -0,0 +1,15 @@ +#pragma once + +static bool success_(OSStatus stat, const char *func, const char *call) +{ + if (stat != noErr) { + blog(LOG_WARNING, "%s: %s failed: %d", + func, call, (int)stat); + return false; + } + + return true; +} + +#define success(stat, call) \ + success_(stat, __FUNCTION__, call) diff --git a/libobs/audio-monitoring/win32/wasapi-enum-devices.c b/libobs/audio-monitoring/win32/wasapi-enum-devices.c new file mode 100644 index 0000000000000..98ed93a6cb1b6 --- /dev/null +++ b/libobs/audio-monitoring/win32/wasapi-enum-devices.c @@ -0,0 +1,105 @@ +#include "../../obs-internal.h" + +#include "wasapi-output.h" + +#include + +#ifdef __MINGW32__ + +#ifdef DEFINE_PROPERTYKEY +#undef DEFINE_PROPERTYKEY +#endif +#define DEFINE_PROPERTYKEY(id, a, b, c, d, e, f, g, h, i, j, k, l) \ + const PROPERTYKEY id = { { a,b,c, { d,e,f,g,h,i,j,k, } }, l }; +DEFINE_PROPERTYKEY(PKEY_Device_FriendlyName, \ + 0xa45c254e, 0xdf1c, 0x4efd, 0x80, \ + 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0, 14); + +#else + +#include + +#endif + +static bool get_device_info(obs_enum_audio_device_cb cb, void *data, + IMMDeviceCollection *collection, UINT idx) +{ + IPropertyStore *store = NULL; + IMMDevice *device = NULL; + PROPVARIANT name_var; + char utf8_name[512]; + WCHAR *w_id = NULL; + char utf8_id[512]; + bool cont = true; + HRESULT hr; + + hr = collection->lpVtbl->Item(collection, idx, &device); + if (FAILED(hr)) { + goto fail; + } + + hr = device->lpVtbl->GetId(device, &w_id); + if (FAILED(hr)) { + goto fail; + } + + hr = device->lpVtbl->OpenPropertyStore(device, STGM_READ, &store); + if (FAILED(hr)) { + goto fail; + } + + PropVariantInit(&name_var); + hr = store->lpVtbl->GetValue(store, &PKEY_Device_FriendlyName, + &name_var); + if (FAILED(hr)) { + goto fail; + } + + os_wcs_to_utf8(w_id, 0, utf8_id, 512); + os_wcs_to_utf8(name_var.pwszVal, 0, utf8_name, 512); + + cont = cb(data, utf8_name, utf8_id); + +fail: + safe_release(store); + safe_release(device); + if (w_id) + CoTaskMemFree(w_id); + return cont; +} + +void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, + void *data) +{ + IMMDeviceEnumerator *enumerator = NULL; + IMMDeviceCollection *collection = NULL; + UINT count; + HRESULT hr; + + hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, + &IID_IMMDeviceEnumerator, &enumerator); + if (FAILED(hr)) { + goto fail; + } + + hr = enumerator->lpVtbl->EnumAudioEndpoints(enumerator, eRender, + DEVICE_STATE_ACTIVE, &collection); + if (FAILED(hr)) { + goto fail; + } + + hr = collection->lpVtbl->GetCount(collection, &count); + if (FAILED(hr)) { + goto fail; + } + + for (UINT i = 0; i < count; i++) { + if (!get_device_info(cb, data, collection, i)) { + break; + } + } + +fail: + safe_release(enumerator); + safe_release(collection); +} diff --git a/libobs/audio-monitoring/win32/wasapi-output.c b/libobs/audio-monitoring/win32/wasapi-output.c new file mode 100644 index 0000000000000..1145f8a007e5f --- /dev/null +++ b/libobs/audio-monitoring/win32/wasapi-output.c @@ -0,0 +1,402 @@ +#include "../../media-io/audio-resampler.h" +#include "../../util/circlebuf.h" +#include "../../util/platform.h" +#include "../../util/darray.h" +#include "../../obs-internal.h" + +#include "wasapi-output.h" + +#define ACTUALLY_DEFINE_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \ + EXTERN_C const GUID DECLSPEC_SELECTANY name \ + = { l, w1, w2, { b1, b2, b3, b4, b5, b6, b7, b8 } } + +ACTUALLY_DEFINE_GUID(CLSID_MMDeviceEnumerator, + 0xBCDE0395, 0xE52F, 0x467C, + 0x8E, 0x3D, 0xC4, 0x57, 0x92, 0x91, 0x69, 0x2E); +ACTUALLY_DEFINE_GUID(IID_IMMDeviceEnumerator, + 0xA95664D2, 0x9614, 0x4F35, + 0xA7, 0x46, 0xDE, 0x8D, 0xB6, 0x36, 0x17, 0xE6); +ACTUALLY_DEFINE_GUID(IID_IAudioClient, + 0x1CB9AD4C, 0xDBFA, 0x4C32, + 0xB1, 0x78, 0xC2, 0xF5, 0x68, 0xA7, 0x03, 0xB2); +ACTUALLY_DEFINE_GUID(IID_IAudioRenderClient, + 0xF294ACFC, 0x3146, 0x4483, + 0xA7, 0xBF, 0xAD, 0xDC, 0xA7, 0xC2, 0x60, 0xE2); + +struct audio_monitor { + obs_source_t *source; + IMMDevice *device; + IAudioClient *client; + IAudioRenderClient *render; + + uint64_t last_recv_time; + audio_resampler_t *resampler; + uint32_t sample_rate; + uint32_t channels; + bool source_has_video : 1; + + int64_t lowest_audio_offset; + struct circlebuf delay_buffer; + uint32_t delay_size; + + DARRAY(float) buf; + pthread_mutex_t playback_mutex; +}; + +/* #define DEBUG_AUDIO */ + +static bool process_audio_delay(struct audio_monitor *monitor, + float **data, uint32_t *frames, uint64_t ts, uint32_t pad) +{ + obs_source_t *s = monitor->source; + uint64_t last_frame_ts = s->last_frame_ts; + uint64_t cur_time = os_gettime_ns(); + uint64_t front_ts; + uint64_t cur_ts; + int64_t diff; + uint32_t blocksize = monitor->channels * sizeof(float); + + /* cut off audio if long-since leftover audio in delay buffer */ + if (cur_time - monitor->last_recv_time > 1000000000) + circlebuf_free(&monitor->delay_buffer); + monitor->last_recv_time = cur_time; + + circlebuf_push_back(&monitor->delay_buffer, &ts, sizeof(ts)); + circlebuf_push_back(&monitor->delay_buffer, frames, sizeof(*frames)); + circlebuf_push_back(&monitor->delay_buffer, *data, + *frames * blocksize); + + while (monitor->delay_buffer.size != 0) { + size_t size; + bool bad_diff; + + circlebuf_peek_front(&monitor->delay_buffer, &cur_ts, + sizeof(ts)); + front_ts = cur_ts - + ((uint64_t)pad * 1000000000ULL / + (uint64_t)monitor->sample_rate); + diff = (int64_t)front_ts - (int64_t)last_frame_ts; + bad_diff = llabs(diff) > 5000000000; + + /* delay audio if rushing */ + if (!bad_diff && diff > 75000000) { +#ifdef DEBUG_AUDIO + blog(LOG_INFO, "audio rushing, cutting audio, " + "diff: %lld, delay buffer size: %lu, " + "v: %llu: a: %llu", + diff, (int)monitor->delay_buffer.size, + last_frame_ts, front_ts); +#endif + return false; + } + + circlebuf_pop_front(&monitor->delay_buffer, NULL, sizeof(ts)); + circlebuf_pop_front(&monitor->delay_buffer, frames, + sizeof(*frames)); + + size = *frames * blocksize; + da_resize(monitor->buf, size); + circlebuf_pop_front(&monitor->delay_buffer, + monitor->buf.array, size); + + /* cut audio if dragging */ + if (!bad_diff && diff < -75000000 && monitor->delay_buffer.size > 0) { +#ifdef DEBUG_AUDIO + blog(LOG_INFO, "audio dragging, cutting audio, " + "diff: %lld, delay buffer size: %lu, " + "v: %llu: a: %llu", + diff, (int)monitor->delay_buffer.size, + last_frame_ts, front_ts); +#endif + continue; + } + + *data = monitor->buf.array; + return true; + } + + return false; +} + +static void on_audio_playback(void *param, obs_source_t *source, + const struct audio_data *audio_data, bool muted) +{ + struct audio_monitor *monitor = param; + IAudioRenderClient *render = monitor->render; + uint8_t *resample_data[MAX_AV_PLANES]; + float vol = source->user_volume; + uint32_t resample_frames; + uint64_t ts_offset; + bool success; + BYTE *output; + + if (pthread_mutex_trylock(&monitor->playback_mutex) != 0) { + return; + } + if (os_atomic_load_long(&source->activate_refs) == 0) { + goto unlock; + } + + success = audio_resampler_resample(monitor->resampler, resample_data, + &resample_frames, &ts_offset, + (const uint8_t *const *)audio_data->data, + (uint32_t)audio_data->frames); + if (!success) { + goto unlock; + } + + UINT32 pad = 0; + monitor->client->lpVtbl->GetCurrentPadding(monitor->client, &pad); + + if (monitor->source_has_video) { + uint64_t ts = audio_data->timestamp - ts_offset; + + if (!process_audio_delay(monitor, (float**)(&resample_data[0]), + &resample_frames, ts, pad)) { + goto unlock; + } + } + + HRESULT hr = render->lpVtbl->GetBuffer(render, resample_frames, + &output); + if (FAILED(hr)) { + goto unlock; + } + + if (!muted) { + /* apply volume */ + if (!close_float(vol, 1.0f, EPSILON)) { + register float *cur = (float*)resample_data[0]; + register float *end = cur + + resample_frames * monitor->channels; + + while (cur < end) + *(cur++) *= vol; + } + memcpy(output, resample_data[0], + resample_frames * monitor->channels * + sizeof(float)); + } + + render->lpVtbl->ReleaseBuffer(render, resample_frames, + muted ? AUDCLNT_BUFFERFLAGS_SILENT : 0); + +unlock: + pthread_mutex_unlock(&monitor->playback_mutex); +} + +static inline void audio_monitor_free(struct audio_monitor *monitor) +{ + if (monitor->source) { + obs_source_remove_audio_capture_callback( + monitor->source, on_audio_playback, monitor); + } + + if (monitor->client) + monitor->client->lpVtbl->Stop(monitor->client); + + safe_release(monitor->device); + safe_release(monitor->client); + safe_release(monitor->render); + audio_resampler_destroy(monitor->resampler); + circlebuf_free(&monitor->delay_buffer); + da_free(monitor->buf); +} + +static enum speaker_layout convert_speaker_layout(DWORD layout, WORD channels) +{ + switch (layout) { + case KSAUDIO_SPEAKER_QUAD: return SPEAKERS_QUAD; + case KSAUDIO_SPEAKER_2POINT1: return SPEAKERS_2POINT1; + case KSAUDIO_SPEAKER_4POINT1: return SPEAKERS_4POINT1; + case KSAUDIO_SPEAKER_SURROUND: return SPEAKERS_SURROUND; + case KSAUDIO_SPEAKER_5POINT1: return SPEAKERS_5POINT1; + case KSAUDIO_SPEAKER_5POINT1_SURROUND: return SPEAKERS_5POINT1_SURROUND; + case KSAUDIO_SPEAKER_7POINT1: return SPEAKERS_7POINT1; + case KSAUDIO_SPEAKER_7POINT1_SURROUND: return SPEAKERS_7POINT1_SURROUND; + } + + return (enum speaker_layout)channels; +} + +static bool audio_monitor_init(struct audio_monitor *monitor) +{ + IMMDeviceEnumerator *immde = NULL; + WAVEFORMATEX *wfex = NULL; + bool success = false; + UINT32 frames; + HRESULT hr; + + const char *id = obs->audio.monitoring_device_id; + if (!id) { + return false; + } + + pthread_mutex_init_value(&monitor->playback_mutex); + + /* ------------------------------------------ * + * Init device */ + + hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, + &IID_IMMDeviceEnumerator, (void**)&immde); + if (FAILED(hr)) { + return false; + } + + if (strcmp(id, "default") == 0) { + hr = immde->lpVtbl->GetDefaultAudioEndpoint(immde, + eRender, eConsole, &monitor->device); + } else { + wchar_t w_id[512]; + os_utf8_to_wcs(id, 0, w_id, 512); + + hr = immde->lpVtbl->GetDevice(immde, w_id, &monitor->device); + } + + if (FAILED(hr)) { + goto fail; + } + + /* ------------------------------------------ * + * Init client */ + + hr = monitor->device->lpVtbl->Activate(monitor->device, + &IID_IAudioClient, CLSCTX_ALL, NULL, + (void**)&monitor->client); + if (FAILED(hr)) { + goto fail; + } + + hr = monitor->client->lpVtbl->GetMixFormat(monitor->client, &wfex); + if (FAILED(hr)) { + goto fail; + } + + hr = monitor->client->lpVtbl->Initialize(monitor->client, + AUDCLNT_SHAREMODE_SHARED, 0, + 10000000, 0, wfex, NULL); + if (FAILED(hr)) { + goto fail; + } + + /* ------------------------------------------ * + * Init resampler */ + + const struct audio_output_info *info = audio_output_get_info( + obs->audio.audio); + WAVEFORMATEXTENSIBLE *ext = (WAVEFORMATEXTENSIBLE*)wfex; + struct resample_info from; + struct resample_info to; + + from.samples_per_sec = info->samples_per_sec; + from.speakers = info->speakers; + from.format = AUDIO_FORMAT_FLOAT_PLANAR; + + to.samples_per_sec = (uint32_t)wfex->nSamplesPerSec; + to.speakers = convert_speaker_layout(ext->dwChannelMask, + wfex->nChannels); + to.format = AUDIO_FORMAT_FLOAT; + + monitor->sample_rate = (uint32_t)wfex->nSamplesPerSec; + monitor->channels = wfex->nChannels; + monitor->resampler = audio_resampler_create(&to, &from); + if (!monitor->resampler) { + goto fail; + } + + /* ------------------------------------------ * + * Init client */ + + hr = monitor->client->lpVtbl->GetBufferSize(monitor->client, &frames); + if (FAILED(hr)) { + goto fail; + } + + hr = monitor->client->lpVtbl->GetService(monitor->client, + &IID_IAudioRenderClient, (void**)&monitor->render); + if (FAILED(hr)) { + goto fail; + } + + if (pthread_mutex_init(&monitor->playback_mutex, NULL) != 0) { + goto fail; + } + + hr = monitor->client->lpVtbl->Start(monitor->client); + if (FAILED(hr)) { + goto fail; + } + + success = true; + +fail: + safe_release(immde); + if (wfex) + CoTaskMemFree(wfex); + return success; +} + +static void audio_monitor_init_final(struct audio_monitor *monitor, + obs_source_t *source) +{ + monitor->source = source; + monitor->source_has_video = + (source->info.output_flags & OBS_SOURCE_VIDEO) != 0; + obs_source_add_audio_capture_callback(source, on_audio_playback, + monitor); +} + +struct audio_monitor *audio_monitor_create(obs_source_t *source) +{ + struct audio_monitor monitor = {0}; + struct audio_monitor *out; + + if (!audio_monitor_init(&monitor)) { + goto fail; + } + + out = bmemdup(&monitor, sizeof(monitor)); + + pthread_mutex_lock(&obs->audio.monitoring_mutex); + da_push_back(obs->audio.monitors, &out); + pthread_mutex_unlock(&obs->audio.monitoring_mutex); + + audio_monitor_init_final(out, source); + return out; + +fail: + audio_monitor_free(&monitor); + return NULL; +} + +void audio_monitor_reset(struct audio_monitor *monitor) +{ + struct audio_monitor new_monitor = {0}; + bool success; + + pthread_mutex_lock(&monitor->playback_mutex); + success = audio_monitor_init(&new_monitor); + pthread_mutex_unlock(&monitor->playback_mutex); + + if (success) { + obs_source_t *source = monitor->source; + audio_monitor_free(monitor); + *monitor = new_monitor; + audio_monitor_init_final(monitor, source); + } else { + audio_monitor_free(&new_monitor); + } +} + +void audio_monitor_destroy(struct audio_monitor *monitor) +{ + if (monitor) { + audio_monitor_free(monitor); + + pthread_mutex_lock(&obs->audio.monitoring_mutex); + da_erase_item(obs->audio.monitors, &monitor); + pthread_mutex_unlock(&obs->audio.monitoring_mutex); + + bfree(monitor); + } +} diff --git a/libobs/audio-monitoring/win32/wasapi-output.h b/libobs/audio-monitoring/win32/wasapi-output.h new file mode 100644 index 0000000000000..d43a418838f3f --- /dev/null +++ b/libobs/audio-monitoring/win32/wasapi-output.h @@ -0,0 +1,13 @@ +#include +#include +#include + +#define KSAUDIO_SPEAKER_4POINT1 (KSAUDIO_SPEAKER_QUAD|SPEAKER_LOW_FREQUENCY) +#define KSAUDIO_SPEAKER_2POINT1 (KSAUDIO_SPEAKER_STEREO|SPEAKER_LOW_FREQUENCY) + +#define safe_release(ptr) \ + do { \ + if (ptr) { \ + ptr->lpVtbl->Release(ptr); \ + } \ + } while (false) diff --git a/libobs/obs-internal.h b/libobs/obs-internal.h index 470a1090456dc..f6f2450b9e825 100644 --- a/libobs/obs-internal.h +++ b/libobs/obs-internal.h @@ -277,8 +277,9 @@ struct obs_core_video { gs_effect_t *deinterlace_yadif_2x_effect; }; +struct audio_monitor; + struct obs_core_audio { - /* TODO: sound output subsystem */ audio_t *audio; DARRAY(struct obs_source*) render_order; @@ -290,6 +291,11 @@ struct obs_core_audio { int total_buffering_ticks; float user_volume; + + pthread_mutex_t monitoring_mutex; + DARRAY(struct audio_monitor*) monitors; + char *monitoring_device_name; + char *monitoring_device_id; }; /* user sources, output channels, and displays */ @@ -546,6 +552,7 @@ struct obs_source { volatile bool timing_set; volatile uint64_t timing_adjust; uint64_t resample_offset; + uint64_t last_audio_ts; uint64_t next_audio_ts_min; uint64_t next_audio_sys_ts_min; uint64_t last_frame_ts; @@ -661,6 +668,9 @@ struct obs_source { enum obs_transition_mode transition_mode; enum obs_transition_scale_type transition_scale_type; struct matrix4 transition_matrices[2]; + + struct audio_monitor *monitor; + enum obs_monitoring_type monitoring_type; }; extern const struct obs_source_info *get_source_info(const char *id); @@ -679,6 +689,10 @@ extern void obs_transition_enum_sources(obs_source_t *transition, extern void obs_transition_save(obs_source_t *source, obs_data_t *data); extern void obs_transition_load(obs_source_t *source, obs_data_t *data); +struct audio_monitor *audio_monitor_create(obs_source_t *source); +void audio_monitor_reset(struct audio_monitor *monitor); +extern void audio_monitor_destroy(struct audio_monitor *monitor); + extern void obs_source_destroy(struct obs_source *source); enum view_type { diff --git a/libobs/obs-source.c b/libobs/obs-source.c index ab472c0da3f66..11f3cc8f1df5a 100644 --- a/libobs/obs-source.c +++ b/libobs/obs-source.c @@ -503,6 +503,8 @@ void obs_source_destroy(struct obs_source *source) source->context.data = NULL; } + audio_monitor_destroy(source->monitor); + obs_hotkey_unregister(source->push_to_talk_key); obs_hotkey_unregister(source->push_to_mute_key); obs_hotkey_pair_unregister(source->mute_unmute_key); @@ -1184,6 +1186,7 @@ static void source_output_audio_data(obs_source_t *source, in.timestamp = source->next_audio_ts_min; } + source->last_audio_ts = in.timestamp; source->next_audio_ts_min = in.timestamp + conv_frames_to_time(sample_rate, in.frames); @@ -1226,10 +1229,12 @@ static void source_output_audio_data(obs_source_t *source, source->last_sync_offset = sync_offset; } - if (push_back && source->audio_ts) - source_output_audio_push_back(source, &in); - else - source_output_audio_place(source, &in); + if (source->monitoring_type != OBS_MONITORING_TYPE_MONITOR_ONLY) { + if (push_back && source->audio_ts) + source_output_audio_push_back(source, &in); + else + source_output_audio_place(source, &in); + } pthread_mutex_unlock(&source->audio_buf_mutex); @@ -3891,3 +3896,38 @@ void obs_source_remove_audio_capture_callback(obs_source_t *source, da_erase_item(source->audio_cb_list, &info); pthread_mutex_unlock(&source->audio_cb_mutex); } + +void obs_source_set_monitoring_type(obs_source_t *source, + enum obs_monitoring_type type) +{ + bool was_on; + bool now_on; + + if (!obs_source_valid(source, "obs_source_set_monitoring_type")) + return; + if (source->info.output_flags & OBS_SOURCE_DO_NOT_MONITOR) + return; + if (source->monitoring_type == type) + return; + + was_on = source->monitoring_type != OBS_MONITORING_TYPE_NONE; + now_on = type != OBS_MONITORING_TYPE_NONE; + + if (was_on != now_on) { + if (!was_on) { + source->monitor = audio_monitor_create(source); + } else { + audio_monitor_destroy(source->monitor); + source->monitor = NULL; + } + } + + source->monitoring_type = type; +} + +enum obs_monitoring_type obs_source_get_monitoring_type( + const obs_source_t *source) +{ + return obs_source_valid(source, "obs_source_get_monitoring_type") ? + source->monitoring_type : OBS_MONITORING_TYPE_NONE; +} diff --git a/libobs/obs-source.h b/libobs/obs-source.h index dfd4a76ec0dda..bb3ae3cbc68f0 100644 --- a/libobs/obs-source.h +++ b/libobs/obs-source.h @@ -120,6 +120,14 @@ enum obs_source_type { */ #define OBS_SOURCE_DEPRECATED (1<<8) +/** + * Source cannot have its audio monitored + * + * Specifies that this source may cause a feedback loop if audio is monitored. + * This is used primarily with desktop audio capture sources. + */ +#define OBS_SOURCE_DO_NOT_MONITOR (1<<9) + /** @} */ typedef void (*obs_source_enum_proc_t)(obs_source_t *parent, diff --git a/libobs/obs.c b/libobs/obs.c index 685d892075027..53e2fefc15c95 100644 --- a/libobs/obs.c +++ b/libobs/obs.c @@ -488,10 +488,22 @@ static bool obs_init_audio(struct audio_output_info *ai) struct obs_core_audio *audio = &obs->audio; int errorcode; - /* TODO: sound subsystem */ + pthread_mutexattr_t attr; + + pthread_mutex_init_value(&audio->monitoring_mutex); + + if (pthread_mutexattr_init(&attr) != 0) + return false; + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0) + return false; + if (pthread_mutex_init(&audio->monitoring_mutex, &attr) != 0) + return false; audio->user_volume = 1.0f; + audio->monitoring_device_name = bstrdup("Default"); + audio->monitoring_device_id = bstrdup("default"); + errorcode = audio_output_open(&audio->audio, ai); if (errorcode == AUDIO_OUTPUT_SUCCESS) return true; @@ -513,6 +525,11 @@ static void obs_free_audio(void) da_free(audio->render_order); da_free(audio->root_nodes); + da_free(audio->monitors); + bfree(audio->monitoring_device_name); + bfree(audio->monitoring_device_id); + pthread_mutex_destroy(&audio->monitoring_mutex); + memset(audio, 0, sizeof(struct obs_core_audio)); } @@ -725,6 +742,8 @@ static bool obs_init(const char *locale, const char *module_config_path, { obs = bzalloc(sizeof(struct obs_core)); + pthread_mutex_init_value(&obs->audio.monitoring_mutex); + obs->name_store_owned = !store; obs->name_store = store ? store : profiler_name_store_create(); if (!obs->name_store) { @@ -1454,6 +1473,7 @@ static obs_source_t *obs_load_source_type(obs_data_t *source_data) uint32_t mixers; int di_order; int di_mode; + int monitoring_type; source = obs_source_create(id, name, settings, hotkeys); @@ -1505,6 +1525,10 @@ static obs_source_t *obs_load_source_type(obs_data_t *source_data) obs_source_set_deinterlace_field_order(source, (enum obs_deinterlace_field_order)di_order); + monitoring_type = (int)obs_data_get_int(source_data, "monitoring_type"); + obs_source_set_monitoring_type(source, + (enum obs_monitoring_type)monitoring_type); + if (filters) { size_t count = obs_data_array_count(filters); @@ -1601,6 +1625,7 @@ obs_data_t *obs_save_source(obs_source_t *source) uint64_t ptm_delay = obs_source_get_push_to_mute_delay(source); bool push_to_talk= obs_source_push_to_talk_enabled(source); uint64_t ptt_delay = obs_source_get_push_to_talk_delay(source); + int m_type = (int)obs_source_get_monitoring_type(source); int di_mode = (int)obs_source_get_deinterlace_mode(source); int di_order = (int)obs_source_get_deinterlace_field_order(source); @@ -1630,6 +1655,7 @@ obs_data_t *obs_save_source(obs_source_t *source) obs_data_set_obj (source_data, "hotkeys", hotkey_data); obs_data_set_int (source_data, "deinterlace_mode", di_mode); obs_data_set_int (source_data, "deinterlace_field_order", di_order); + obs_data_set_int (source_data, "monitoring_type", m_type); if (source->info.type == OBS_SOURCE_TYPE_TRANSITION) obs_transition_save(source, source_data); @@ -1876,3 +1902,45 @@ bool obs_obj_invalid(void *obj) return !context->data; } + +bool obs_set_audio_monitoring_device(const char *name, const char *id) +{ + if (!obs || !name || !id || !*name || !*id) + return false; + +#ifdef _WIN32 + pthread_mutex_lock(&obs->audio.monitoring_mutex); + + if (strcmp(id, obs->audio.monitoring_device_id) == 0) + return true; + + if (obs->audio.monitoring_device_name) + bfree(obs->audio.monitoring_device_name); + if (obs->audio.monitoring_device_id) + bfree(obs->audio.monitoring_device_id); + + obs->audio.monitoring_device_name = bstrdup(name); + obs->audio.monitoring_device_id = bstrdup(id); + + for (size_t i = 0; i < obs->audio.monitors.num; i++) { + struct audio_monitor *monitor = obs->audio.monitors.array[i]; + audio_monitor_reset(monitor); + } + + pthread_mutex_unlock(&obs->audio.monitoring_mutex); + return true; +#else + return false; +#endif +} + +void obs_get_audio_monitoring_device(const char **name, const char **id) +{ + if (!obs) + return; + + if (name) + *name = obs->audio.monitoring_device_name; + if (id) + *id = obs->audio.monitoring_device_id; +} diff --git a/libobs/obs.h b/libobs/obs.h index 4c4cc8727c6a8..cf9cd366fea30 100644 --- a/libobs/obs.h +++ b/libobs/obs.h @@ -582,6 +582,15 @@ EXPORT enum obs_obj_type obs_obj_get_type(void *obj); EXPORT const char *obs_obj_get_id(void *obj); EXPORT bool obs_obj_invalid(void *obj); +typedef bool (*obs_enum_audio_device_cb)(void *data, const char *name, + const char *id); + +EXPORT void obs_enum_audio_monitoring_devices(obs_enum_audio_device_cb cb, + void *data); + +EXPORT bool obs_set_audio_monitoring_device(const char *name, const char *id); +EXPORT void obs_get_audio_monitoring_device(const char **name, const char **id); + /* ------------------------------------------------------------------------- */ /* View context */ @@ -914,6 +923,17 @@ EXPORT void obs_source_set_deinterlace_field_order(obs_source_t *source, EXPORT enum obs_deinterlace_field_order obs_source_get_deinterlace_field_order( const obs_source_t *source); +enum obs_monitoring_type { + OBS_MONITORING_TYPE_NONE, + OBS_MONITORING_TYPE_MONITOR_ONLY, + OBS_MONITORING_TYPE_MONITOR_AND_OUTPUT +}; + +EXPORT void obs_source_set_monitoring_type(obs_source_t *source, + enum obs_monitoring_type type); +EXPORT enum obs_monitoring_type obs_source_get_monitoring_type( + const obs_source_t *source); + /* ------------------------------------------------------------------------- */ /* Functions used by sources */