diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index c171e94a..a761f4c0 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -50,6 +50,7 @@ add_subdirectory(dependencies/jni-voithos) file(GLOB SOURCES_ROOT "src/*.cpp") file(GLOB SOURCES_API "src/api/*.cpp") +file(GLOB SOURCES_API_AUDIO "src/api/audio/*.cpp") file(GLOB SOURCES_MEDIA "src/media/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO "src/media/audio/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO_OS "src/media/audio/${SOURCE_TARGET}/*.cpp") @@ -63,6 +64,7 @@ file(GLOB SOURCES_RTC "src/rtc/*.cpp") list(APPEND SOURCES ${SOURCES_ROOT} ${SOURCES_API} + ${SOURCES_API_AUDIO} ${SOURCES_MEDIA} ${SOURCES_MEDIA_AUDIO} ${SOURCES_MEDIA_AUDIO_OS} diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt index b4b83696..a11a7b5e 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt @@ -237,14 +237,16 @@ if (PATCHES) message(STATUS "Applying ${PATCH}") execute_process( - COMMAND patch -p0 --forward - WORKING_DIRECTORY "${WEBRTC_SRC_DIR}" + COMMAND git apply + WORKING_DIRECTORY "${WEBRTC_SRC}" INPUT_FILE "${PATCH}" OUTPUT_VARIABLE OUTPUT RESULT_VARIABLE RESULT) if (RESULT EQUAL 0) message(STATUS "Patch applied: ${PATCH}") + else() + message(STATUS "Warning: Patch ${PATCH} not applied ....") endif() endforeach(PATCH) endif() diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/macos/patch_for_mac_deviceId.patch b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/macos/patch_for_mac_deviceId.patch new file mode 100644 index 00000000..af35cefb --- /dev/null +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/macos/patch_for_mac_deviceId.patch @@ -0,0 +1,32 @@ +Subject: [PATCH] patch for mac deviceId +--- +Index: modules/audio_device/mac/audio_device_mac.cc +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc +--- a/modules/audio_device/mac/audio_device_mac.cc (revision e4445e46a910eb407571ec0b0b8b7043562678cf) ++++ b/modules/audio_device/mac/audio_device_mac.cc (date 1757498230183) +@@ -834,6 +834,10 @@ + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); ++ AudioDeviceID deviceIds[MaxNumberDevices]; ++ int numberDevices = GetNumberDevices(kAudioDevicePropertyScopeOutput, deviceIds, MaxNumberDevices); ++ std::string deviceId = std::to_string(deviceIds[index]); ++ deviceId.copy(guid, kAdmMaxGuidSize); + } + + return GetDeviceName(kAudioDevicePropertyScopeOutput, index, +@@ -853,6 +857,10 @@ + + if (guid != NULL) { + memset(guid, 0, kAdmMaxGuidSize); ++ AudioDeviceID deviceIds[MaxNumberDevices]; ++ int numberDevices = GetNumberDevices(kAudioDevicePropertyScopeInput, deviceIds, MaxNumberDevices); ++ std::string deviceId = std::to_string(deviceIds[index]); ++ deviceId.copy(guid, kAdmMaxGuidSize); + } + + return GetDeviceName(kAudioDevicePropertyScopeInput, index, diff --git a/webrtc-jni/src/main/cpp/include/media/Device.h b/webrtc-jni/src/main/cpp/include/media/Device.h index f51d5014..a8f5896f 100644 --- a/webrtc-jni/src/main/cpp/include/media/Device.h +++ b/webrtc-jni/src/main/cpp/include/media/Device.h @@ -28,6 +28,24 @@ namespace jni { namespace avdev { + /** + DeviceTransport and DeviceFormFactor only for audio devices. + */ + enum class DeviceTransport { + trUnknown, + trHdmi, + trUsb, + trWireless, + }; + + enum class DeviceFormFactor { + ffUnknown, + ffSpeaker, + ffMicrophone, + ffHeadset, + ffHeadphone + }; + class Device { public: @@ -39,6 +57,10 @@ namespace jni std::string getName() const; std::string getDescriptor() const; + DeviceTransport getDeviceTransport(); + DeviceFormFactor getDeviceFormFactor(); + void setDeviceTransport(DeviceTransport deviceTransport); + void setDeviceFormFactor(DeviceFormFactor deviceFormFactor); protected: Device(std::string name, std::string descriptor); @@ -46,6 +68,8 @@ namespace jni private: const std::string name; const std::string descriptor; + DeviceTransport deviceTransport; + DeviceFormFactor deviceFormFactor; }; @@ -63,6 +87,8 @@ namespace jni jmethodID ctor; jfieldID name; jfieldID descriptor; + jfieldID deviceTransport; + jfieldID deviceFormFactor; }; JavaLocalRef toJavaDevice(JNIEnv * env, avdev::DevicePtr device); diff --git a/webrtc-jni/src/main/cpp/include/media/DeviceList.h b/webrtc-jni/src/main/cpp/include/media/DeviceList.h index 8882396d..b6b91786 100644 --- a/webrtc-jni/src/main/cpp/include/media/DeviceList.h +++ b/webrtc-jni/src/main/cpp/include/media/DeviceList.h @@ -94,6 +94,13 @@ namespace jni return devicesSet.empty(); } + void clearDevices() + { + std::unique_lock mlock(mutex); + devicesSet.clear(); + mlock.unlock(); + } + std::set devices() { std::unique_lock mlock(mutex); diff --git a/webrtc-jni/src/main/cpp/include/media/audio/AudioDevice.h b/webrtc-jni/src/main/cpp/include/media/audio/AudioDevice.h index c91b7530..2edf4873 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/AudioDevice.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/AudioDevice.h @@ -27,11 +27,19 @@ namespace jni { namespace avdev { + + enum class AudioDeviceDirectionType { + adtUnknown, + adtCapture, + adtRender + }; + class AudioDevice : public Device { public: AudioDevice(std::string name, std::string descriptor); virtual ~AudioDevice() {}; + AudioDeviceDirectionType directionType; }; } @@ -46,6 +54,9 @@ namespace jni jmethodID ctor; jfieldID name; jfieldID descriptor; + jfieldID deviceTransport; + jfieldID deviceFormFactor; + jfieldID directionType; }; JavaLocalRef toJavaAudioDevice(JNIEnv * env, avdev::DevicePtr device); diff --git a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioDeviceManager.h b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioDeviceManager.h index 24ff3e15..b07b429b 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioDeviceManager.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/linux/PulseAudioDeviceManager.h @@ -54,8 +54,12 @@ namespace jni static void getSinkCallback(pa_context * ctx, const pa_sink_info * info, int last, void * userdata); static void newSinkCallback(pa_context * ctx, const pa_sink_info * info, int last, void * userdata); - void insertDevice(DeviceList & devices, const char * name, const char * desc, uint32_t index, bool notify); - void removeDevice(DeviceList & devices, uint32_t index); + void insertDevice(DeviceList & devices, pa_proplist * proplist, const char * name, const char * desc, uint32_t index, bool notify, bool isCapture); + void removeDevice(DeviceList & devices, uint32_t index, bool isCapture); + + void fillAdditionalTypes(AudioDevicePtr device, pa_proplist * proplist); + DeviceFormFactor getActualFormFactor(std::string formFactor); + DeviceTransport getActualTransport(std::string transport); private: pa_threaded_mainloop * mainloop; diff --git a/webrtc-jni/src/main/cpp/include/media/audio/macos/CoreAudioDeviceManager.h b/webrtc-jni/src/main/cpp/include/media/audio/macos/CoreAudioDeviceManager.h index e1289634..7eb36f26 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/macos/CoreAudioDeviceManager.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/macos/CoreAudioDeviceManager.h @@ -47,6 +47,9 @@ namespace jni bool insertAudioDevice(const AudioDevicePtr & device, const AudioObjectPropertyScope & scope); int getChannelCount(const AudioDeviceID & deviceID, const AudioObjectPropertyScope & scope); AudioDeviceID getDefaultDeviceID(const AudioObjectPropertyScope & scope); + DeviceFormFactor getActualFormFactor(const unsigned int sourceID); + DeviceTransport getActualTransport(const unsigned int transportType); + void fillAdditionalTypes(const AudioDeviceID & deviceID, const AudioObjectPropertyScope & scope, AudioDevicePtr device); static OSStatus deviceListenerProc(AudioObjectID objectID, UInt32 numberAddresses, const AudioObjectPropertyAddress addresses[], void * clientData); }; diff --git a/webrtc-jni/src/main/cpp/include/media/audio/windows/WindowsAudioDeviceManager.h b/webrtc-jni/src/main/cpp/include/media/audio/windows/WindowsAudioDeviceManager.h index dee044db..1e8fcbbe 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/windows/WindowsAudioDeviceManager.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/windows/WindowsAudioDeviceManager.h @@ -17,6 +17,7 @@ #ifndef JNI_WEBRTC_MEDIA_MF_AUDIO_DEVICE_MANAGER_H_ #define JNI_WEBRTC_MEDIA_MF_AUDIO_DEVICE_MANAGER_H_ +#include #include "media/audio/AudioDeviceManager.h" #include "platform/windows/ComPtr.h" #include "platform/windows/ComInitializer.h" @@ -46,7 +47,10 @@ namespace jni AudioDevicePtr createDefaultAudioDevice(const EDataFlow & dataFlow); AudioDevicePtr createAudioDevice(LPCWSTR deviceId, EDataFlow * dataFlow); bool insertAudioDevice(AudioDevicePtr device, EDataFlow dataFlow); - void removeAudioDevice(DeviceList & devices, std::string id); + void removeAudioDevice(DeviceList & devices, std::string id, EDataFlow dataFlow); + DeviceFormFactor getActualFormFactor(EndpointFormFactor formFactor); + DeviceTransport getActualTransport(EndpointFormFactor formFactor); + void fillAdditionalTypes(AudioDevicePtr device); // IMMNotificationClient implementation. STDMETHOD_(ULONG, AddRef)(); diff --git a/webrtc-jni/src/main/cpp/include/platform/windows/WinUtils.h b/webrtc-jni/src/main/cpp/include/platform/windows/WinUtils.h index 059649c9..47e87f47 100644 --- a/webrtc-jni/src/main/cpp/include/platform/windows/WinUtils.h +++ b/webrtc-jni/src/main/cpp/include/platform/windows/WinUtils.h @@ -102,4 +102,34 @@ inline std::string RoleToStr(ERole role) } } +inline std::string FormFactorToStr(EndpointFormFactor formFactor) +{ + switch (formFactor) { + case RemoteNetworkDevice: + return "RemoteNetworkDevice"; + case Speakers: + return "Speakers"; + case Headphones: + return "Headphones"; + case Microphone: + return "Microphone"; + case Headset: + return "Headset"; + case Handset: + return "Handset"; + case UnknownDigitalPassthrough: + return "UnknownDigitalPassthrough"; + case SPDIF: + return "SPDIF"; + case DigitalAudioDisplayDevice: + return "DigitalAudioDisplayDevice"; + case UnknownFormFactor: + return "UnknownFormFactor"; + case EndpointFormFactor_enum_count: + return "EndpointFormFactor_enum_count"; + default: + return "Unknown Form Factor"; + } +} + #endif \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp index c35558e5..38b82567 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioDeviceModuleBase.cpp @@ -121,6 +121,7 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_audio_AudioDeviceModuleBa for (int i = 0; i < deviceCount; ++i) { if (audioModule->PlayoutDeviceName(i, name, guid) == 0) { auto device = std::make_shared(name, guid); + device -> directionType = jni::avdev::AudioDeviceDirectionType::adtRender; deviceList.add(jni::AudioDevice::toJavaAudioDevice(env, device)); } @@ -145,6 +146,7 @@ JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_audio_AudioDeviceModuleBa for (int i = 0; i < deviceCount; ++i) { if (audioModule->RecordingDeviceName(i, name, guid) == 0) { auto device = std::make_shared(name, guid); + device -> directionType = jni::avdev::AudioDeviceDirectionType::adtCapture; deviceList.add(jni::AudioDevice::toJavaAudioDevice(env, device)); } diff --git a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp index 990da694..165411e0 100644 --- a/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp +++ b/webrtc-jni/src/main/cpp/src/WebRTCContext.cpp @@ -90,6 +90,9 @@ namespace jni JavaEnums::add(env, PKG_AUDIO"AudioProcessingConfig$Pipeline$DownmixMethod"); JavaEnums::add(env, PKG_AUDIO"AudioProcessingConfig$NoiseSuppression$Level"); JavaEnums::add(env, PKG"RTCStatsType"); + JavaEnums::add(env, PKG_MEDIA"DeviceFormFactor"); + JavaEnums::add(env, PKG_MEDIA"DeviceTransport"); + JavaEnums::add(env, PKG_MEDIA"AudioDeviceDirectionType"); JavaFactories::add(env, PKG_MEDIA"audio/AudioTrackSource"); JavaFactories::add(env, PKG_MEDIA"audio/AudioTrack"); diff --git a/webrtc-jni/src/main/cpp/src/media/Device.cpp b/webrtc-jni/src/main/cpp/src/media/Device.cpp index 09d22518..5156bf3d 100644 --- a/webrtc-jni/src/main/cpp/src/media/Device.cpp +++ b/webrtc-jni/src/main/cpp/src/media/Device.cpp @@ -18,6 +18,7 @@ #include "JavaClasses.h" #include "JavaString.h" #include "JNI_WebRTC.h" +#include "JavaEnums.h" namespace jni { @@ -25,7 +26,9 @@ namespace jni { Device::Device(std::string name, std::string descriptor) : name(name), - descriptor(descriptor) + descriptor(descriptor), + deviceTransport(DeviceTransport::trUnknown), + deviceFormFactor(DeviceFormFactor::ffUnknown) { } @@ -53,6 +56,26 @@ namespace jni { return name; } + + DeviceTransport Device::getDeviceTransport() + { + return deviceTransport; + } + + DeviceFormFactor Device::getDeviceFormFactor() + { + return deviceFormFactor; + } + + void Device::setDeviceTransport(DeviceTransport newDeviceTransport) + { + deviceTransport = newDeviceTransport; + } + + void Device::setDeviceFormFactor(DeviceFormFactor newDeviceFormFactor) + { + deviceFormFactor = newDeviceFormFactor; + } } namespace Device @@ -65,6 +88,12 @@ namespace jni JavaString::toJava(env, device->getName()).get(), JavaString::toJava(env, device->getDescriptor()).get()); + auto deviceTransport = JavaEnums::toJava(env, device->getDeviceTransport()); + env->SetObjectField(obj, javaClass->deviceTransport, deviceTransport.get()); + + auto deviceFormFactor = JavaEnums::toJava(env, device->getDeviceFormFactor()); + env->SetObjectField(obj, javaClass->deviceFormFactor, deviceFormFactor.get()); + return JavaLocalRef(env, obj); } @@ -76,6 +105,8 @@ namespace jni name = GetFieldID(env, cls, "name", STRING_SIG); descriptor = GetFieldID(env, cls, "descriptor", STRING_SIG); + deviceTransport = GetFieldID(env, cls, "deviceTransport", "L" PKG_MEDIA "DeviceTransport;"); + deviceFormFactor = GetFieldID(env, cls, "deviceFormFactor", "L" PKG_MEDIA "DeviceFormFactor;"); } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/audio/AudioDevice.cpp b/webrtc-jni/src/main/cpp/src/media/audio/AudioDevice.cpp index 9069c726..0737d8b8 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/AudioDevice.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/AudioDevice.cpp @@ -19,13 +19,15 @@ #include "JavaClasses.h" #include "JavaString.h" #include "JNI_WebRTC.h" +#include "JavaEnums.h" namespace jni { namespace avdev { AudioDevice::AudioDevice(std::string name, std::string descriptor) : - Device(name, descriptor) + Device(name, descriptor), + directionType(AudioDeviceDirectionType::adtUnknown) { } } @@ -40,6 +42,16 @@ namespace jni JavaString::toJava(env, device->getName()).get(), JavaString::toJava(env, device->getDescriptor()).get()); + auto deviceTransport = JavaEnums::toJava(env, device->getDeviceTransport()); + env->SetObjectField(obj, javaClass->deviceTransport, deviceTransport.get()); + + auto deviceFormFactor = JavaEnums::toJava(env, device->getDeviceFormFactor()); + env->SetObjectField(obj, javaClass->deviceFormFactor, deviceFormFactor.get()); + + auto audioDevice = dynamic_cast(device.get()); + auto directionType = JavaEnums::toJava(env, audioDevice->directionType); + env->SetObjectField(obj, javaClass->directionType, directionType.get()); + return JavaLocalRef(env, obj); } @@ -51,6 +63,9 @@ namespace jni name = GetFieldID(env, cls, "name", STRING_SIG); descriptor = GetFieldID(env, cls, "descriptor", STRING_SIG); + directionType = GetFieldID(env, cls, "directionType", "L" PKG_MEDIA "AudioDeviceDirectionType;"); + deviceTransport = GetFieldID(env, cls, "deviceTransport", "L" PKG_MEDIA "DeviceTransport;"); + deviceFormFactor = GetFieldID(env, cls, "deviceFormFactor", "L" PKG_MEDIA "DeviceFormFactor;"); } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/audio/linux/PulseAudioDeviceManager.cpp b/webrtc-jni/src/main/cpp/src/media/audio/linux/PulseAudioDeviceManager.cpp index 60a06549..dd1db433 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/linux/PulseAudioDeviceManager.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/linux/PulseAudioDeviceManager.cpp @@ -139,7 +139,9 @@ namespace jni if (!pa_threaded_mainloop_in_thread(mainloop)) pa_threaded_mainloop_unlock(mainloop); - return std::make_shared(defaultCaptureDescName, defaultCaptureName); + AudioDevicePtr defaultDevice = std::make_shared(defaultCaptureDescName, defaultCaptureName); + defaultDevice->directionType = AudioDeviceDirectionType::adtCapture; + return defaultDevice; } std::set PulseAudioDeviceManager::getAudioCaptureDevices() @@ -174,7 +176,9 @@ namespace jni if (!pa_threaded_mainloop_in_thread(mainloop)) pa_threaded_mainloop_unlock(mainloop); - return std::make_shared(defaultPlaybackDescName, defaultPlaybackName); + AudioDevicePtr defaultDevice = std::make_shared(defaultPlaybackDescName, defaultPlaybackName); + defaultDevice->directionType = AudioDeviceDirectionType::adtRender; + return defaultDevice; } std::set PulseAudioDeviceManager::getAudioPlaybackDevices() @@ -213,7 +217,7 @@ namespace jni } if (info->monitor_of_sink == PA_INVALID_INDEX) { - engine->insertDevice(engine->captureDevices, info->description, info->name, info->index, false); + engine->insertDevice(engine->captureDevices, info->proplist, info->description, info->name, info->index, false, true); } } @@ -226,7 +230,7 @@ namespace jni return; } - engine->insertDevice(engine->captureDevices, info->description, info->name, info->index, true); + engine->insertDevice(engine->captureDevices, info->proplist, info->description, info->name, info->index, true, true); } void PulseAudioDeviceManager::getSinkInfoCallback(pa_context * ctx, const pa_sink_info * info, int last, void * userdata) @@ -250,7 +254,7 @@ namespace jni return; } - engine->insertDevice(engine->playbackDevices, info->description, info->name, info->index, false); + engine->insertDevice(engine->playbackDevices, info->proplist, info->description, info->name, info->index, false, false); } void PulseAudioDeviceManager::newSinkCallback(pa_context * ctx, const pa_sink_info * info, int last, void * userdata) @@ -262,14 +266,20 @@ namespace jni return; } - engine->insertDevice(engine->playbackDevices, info->description, info->name, info->index, true); + engine->insertDevice(engine->playbackDevices, info->proplist, info->description, info->name, info->index, true, false); } - void PulseAudioDeviceManager::insertDevice(DeviceList & devices, const char * name, const char * desc, uint32_t index, bool notify) + void PulseAudioDeviceManager::insertDevice(DeviceList & devices, pa_proplist * proplist, const char * name, const char * desc, uint32_t index, bool notify, bool isCapture) { auto device = std::make_shared(name, desc); if (devices.insertDevice(device)) { + if (isCapture) { + device->directionType = AudioDeviceDirectionType::adtCapture; + } else { + device->directionType = AudioDeviceDirectionType::adtRender; + } + fillAdditionalTypes(device, proplist); deviceMap[index] = device; } if (notify) { @@ -277,7 +287,7 @@ namespace jni } } - void PulseAudioDeviceManager::removeDevice(DeviceList & devices, uint32_t index) + void PulseAudioDeviceManager::removeDevice(DeviceList & devices, uint32_t index, bool isCapture) { auto it = deviceMap.find(index); if (it == deviceMap.end()) { @@ -285,6 +295,12 @@ namespace jni } if (devices.removeDevice(it->second)) { + if (isCapture) { + it->second->directionType = AudioDeviceDirectionType::adtCapture; + } else { + it->second->directionType = AudioDeviceDirectionType::adtRender; + } + notifyDeviceDisconnected(it->second); deviceMap.erase(it); } @@ -317,7 +333,7 @@ namespace jni op = pa_context_get_source_info_by_index(ctx, idx, newSourceCallback, engine); } if (operation == PA_SUBSCRIPTION_EVENT_REMOVE) { - engine->removeDevice(engine->captureDevices, idx); + engine->removeDevice(engine->captureDevices, idx, true); } } if (facility == PA_SUBSCRIPTION_EVENT_SINK) { @@ -325,7 +341,7 @@ namespace jni op = pa_context_get_sink_info_by_index(ctx, idx, newSinkCallback, engine); } if (operation == PA_SUBSCRIPTION_EVENT_REMOVE) { - engine->removeDevice(engine->playbackDevices, idx); + engine->removeDevice(engine->playbackDevices, idx, false); } } @@ -334,5 +350,56 @@ namespace jni } } + void PulseAudioDeviceManager::fillAdditionalTypes(AudioDevicePtr device, pa_proplist * proplist) { + // all property values see here https://docs.rs/libpulse-sys/latest/libpulse_sys/proplist/ + const char *formFactor; + formFactor = pa_proplist_gets(proplist, PA_PROP_DEVICE_FORM_FACTOR); + std::string formFactorStr = ""; + if (formFactor) { + formFactorStr = std::string(formFactor); + } + + const char *deviceTransport; + deviceTransport = pa_proplist_gets(proplist, PA_PROP_DEVICE_BUS); + std::string deviceTransportStr = ""; + if (deviceTransport) { + deviceTransportStr = std::string(deviceTransport); + } + + const char *propHDMI; + propHDMI = pa_proplist_gets(proplist, PA_PROP_DEVICE_BUS_PATH); + if (propHDMI && std::string(propHDMI).find("_hdmi") != std::string::npos) { + deviceTransportStr = "hdmi"; + } + + device->setDeviceFormFactor(getActualFormFactor(formFactorStr)); + device->setDeviceTransport(getActualTransport(deviceTransportStr)); + } + + DeviceFormFactor PulseAudioDeviceManager::getActualFormFactor(std::string formFactor) { + if (formFactor == "speaker") { + return DeviceFormFactor::ffSpeaker; + } else if (formFactor == "microphone") { + return DeviceFormFactor::ffMicrophone; + } else if (formFactor == "headset") { + return DeviceFormFactor::ffHeadset; + } else if (formFactor == "headphone") { + return DeviceFormFactor::ffHeadphone; + } + + return DeviceFormFactor::ffUnknown; + } + + DeviceTransport PulseAudioDeviceManager::getActualTransport(std::string transport) { + if (transport == "usb") { + return DeviceTransport::trUsb; + } else if (transport == "bluetooth") { + return DeviceTransport::trWireless; + } else if (transport == "hdmi") { + return DeviceTransport::trHdmi; + } + + return DeviceTransport::trUnknown; + } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/audio/macos/CoreAudioDeviceManager.cpp b/webrtc-jni/src/main/cpp/src/media/audio/macos/CoreAudioDeviceManager.cpp index 607fd259..cb6e12ec 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/macos/CoreAudioDeviceManager.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/macos/CoreAudioDeviceManager.cpp @@ -20,6 +20,7 @@ #include "rtc_base/logging.h" #include +#include #if !defined(MAC_OS_VERSION_12_0) || MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_VERSION_12_0 #define kAudioObjectPropertyElementMain kAudioObjectPropertyElementMaster @@ -224,9 +225,11 @@ namespace jni } if (scope == kAudioObjectPropertyScopeInput) { + found->directionType = AudioDeviceDirectionType::adtCapture; setDefaultCaptureDevice(found); } else if (scope == kAudioObjectPropertyScopeOutput) { + found->directionType = AudioDeviceDirectionType::adtRender; setDefaultPlaybackDevice(found); } } @@ -257,6 +260,7 @@ namespace jni if (removed == def) { onDefaultDeviceChanged(scope, captureDevices, def); } + removed->directionType = AudioDeviceDirectionType::adtCapture; } else if (scope == kAudioObjectPropertyScopeOutput) { AudioDevicePtr def = getDefaultAudioPlaybackDevice(); @@ -264,6 +268,7 @@ namespace jni if (removed == def) { onDefaultDeviceChanged(scope, playbackDevices, def); } + removed->directionType = AudioDeviceDirectionType::adtRender; } notifyDeviceDisconnected(removed); @@ -300,10 +305,13 @@ namespace jni if (channels > 0) { if (scope == kAudioObjectPropertyScopeOutput) { device = std::make_shared(name, id); + device-> directionType = AudioDeviceDirectionType::adtRender; } else if (scope == kAudioObjectPropertyScopeInput) { device = std::make_shared(name, id); + device->directionType = AudioDeviceDirectionType::adtCapture; } + fillAdditionalTypes(deviceID, scope, device); } return device; @@ -316,9 +324,11 @@ namespace jni } if (scope == kAudioObjectPropertyScopeOutput) { + device->directionType = AudioDeviceDirectionType::adtRender; return playbackDevices.insertDevice(device); } else if (scope == kAudioObjectPropertyScopeInput) { + device->directionType = AudioDeviceDirectionType::adtCapture; return captureDevices.insertDevice(device); } @@ -404,5 +414,67 @@ namespace jni return noErr; } + + void CoreAudioDeviceManager::fillAdditionalTypes(const AudioDeviceID & deviceID, const AudioObjectPropertyScope & scope, AudioDevicePtr device) { + // Get Transport Type + AudioObjectPropertyAddress address = { + kAudioDevicePropertyTransportType, + scope, + kAudioObjectPropertyElementMaster + }; + unsigned int transportType = 0; + UInt32 size = 0; + size = sizeof(transportType); + OSStatus status = AudioObjectGetPropertyData(deviceID, &address, 0, nullptr, &size, &transportType); + // Get Transport Type -- end + + address.mSelector = kAudioDevicePropertyDataSource; + UInt32 sourceCode = 0; + size = sizeof(UInt32); + status = AudioObjectGetPropertyData(deviceID, &address, 0, nullptr, &size, &sourceCode); + + device->setDeviceTransport(getActualTransport(transportType)); + device->setDeviceFormFactor(getActualFormFactor(sourceCode)); + } + + DeviceFormFactor CoreAudioDeviceManager::getActualFormFactor(const unsigned int sourceID) { + switch(sourceID) { + case kIOAudioOutputPortSubTypeInternalSpeaker: + case kIOAudioOutputPortSubTypeExternalSpeaker: + return DeviceFormFactor::ffSpeaker; + case kIOAudioOutputPortSubTypeHeadphones: + return DeviceFormFactor::ffHeadphone; + case kIOAudioInputPortSubTypeInternalMicrophone: + case kIOAudioInputPortSubTypeExternalMicrophone: + return DeviceFormFactor::ffMicrophone; + case kIOAudioInputPortSubTypeCD: + case kIOAudioOutputPortSubTypeSPDIF: + case kIOAudioOutputPortSubTypeLine: + return DeviceFormFactor::ffUnknown; + default: + return DeviceFormFactor::ffUnknown; + } + } + + DeviceTransport CoreAudioDeviceManager::getActualTransport(const unsigned int transportType) { + switch(transportType) { + case kAudioDeviceTransportTypeUSB: + return DeviceTransport::trUsb; + case kAudioDeviceTransportTypeBluetooth: + return DeviceTransport::trWireless; + case kAudioDeviceTransportTypeHDMI: + return DeviceTransport::trHdmi; + case kAudioDeviceTransportTypeFireWire: + case kAudioDeviceTransportTypeDisplayPort: + case kAudioDeviceTransportTypePCI: + case kAudioDeviceTransportTypeVirtual: + case kAudioDeviceTransportTypeAutoAggregate: + case kAudioDeviceTransportTypeBuiltIn: + case kAudioDeviceTransportTypeUnknown: + return DeviceTransport::trUnknown; + default: + return DeviceTransport::trUnknown; + } + } } } diff --git a/webrtc-jni/src/main/cpp/src/media/audio/windows/WindowsAudioDeviceManager.cpp b/webrtc-jni/src/main/cpp/src/media/audio/windows/WindowsAudioDeviceManager.cpp index ad3fb4aa..705d8dd7 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/windows/WindowsAudioDeviceManager.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/windows/WindowsAudioDeviceManager.cpp @@ -27,6 +27,8 @@ #include "modules/audio_device/include/audio_device.h" #include "rtc_base/logging.h" +#include + namespace jni { namespace avdev @@ -112,28 +114,77 @@ namespace jni if (enumCaptureDevices) { int16_t deviceCount = audioModule->RecordingDevices(); + // fix duplicated item when device reconect to port. The comparator does not always work correctly. + if (deviceCount > 0) { + captureDevices.clearDevices(); + } for (int i = 0; i < deviceCount; ++i) { if (audioModule->RecordingDeviceName(i, name, guid) == 0) { AudioDevicePtr device = std::make_shared(name, guid); - insertAudioDevice(device, EDataFlow::eCapture); + fillAdditionalTypes(device); + + insertAudioDevice(device, EDataFlow::eCapture); } } } if (enumRenderDevices) { int16_t deviceCount = audioModule->PlayoutDevices(); + // Fix duplicated item when device reconect to port. The comparator does not always work correctly. + if (deviceCount > 0) { + playbackDevices.clearDevices(); + } + for (int i = 0; i < deviceCount; ++i) { if (audioModule->PlayoutDeviceName(i, name, guid) == 0) { AudioDevicePtr device = std::make_shared(name, guid); - insertAudioDevice(device, EDataFlow::eRender); + fillAdditionalTypes(device); + + insertAudioDevice(device, EDataFlow::eRender); } } } } + void WindowsAudioDeviceManager::fillAdditionalTypes(AudioDevicePtr device) + { + MFInitializer initializer; + ComPtr pDevice; + ComPtr endpoint; + ComPtr propertyStore; + HRESULT hr; + PROPVARIANT ff; + PropVariantInit(&ff); + + // convert std::string to LPWSTR + wchar_t* wDeviceID=new wchar_t[4096]; + size_t convertedChars = 0; + mbstowcs_s(&convertedChars, wDeviceID, device->getDescriptor().length() + 1, device->getDescriptor().c_str(), _TRUNCATE); + + hr = deviceEnumerator->GetDevice(wDeviceID, &pDevice); + delete[] wDeviceID; + + THROW_IF_FAILED(hr, "Audio Device Manager: Enumerator get device with ID: %S failed", device->getDescriptor().c_str()); + + hr = pDevice->QueryInterface(__uuidof(IMMEndpoint), (void**)&endpoint); + THROW_IF_FAILED(hr, "Audio Device Manager: Device get endpoint failed"); + + hr = pDevice->OpenPropertyStore(STGM_READ, &propertyStore); + THROW_IF_FAILED(hr, "Audio Device Manager: Device open property store failed"); + + hr = propertyStore->GetValue(PKEY_AudioEndpoint_FormFactor, &ff); + THROW_IF_FAILED(hr, "Audio Device Manager: PropertyStore get form factor property name failed"); + + EndpointFormFactor formFactor = static_cast(ff.uintVal); + PropVariantClear(&ff); + + device->setDeviceTransport(getActualTransport(formFactor)); + device->setDeviceFormFactor(getActualFormFactor(formFactor)); + } + void WindowsAudioDeviceManager::addDevice(LPCWSTR deviceId) { if (deviceId == nullptr) { @@ -156,8 +207,8 @@ namespace jni std::string id = WideStrToStr(deviceId); - removeAudioDevice(captureDevices, id); - removeAudioDevice(playbackDevices, id); + removeAudioDevice(captureDevices, id, eCapture); + removeAudioDevice(playbackDevices, id, eRender); } AudioDevicePtr WindowsAudioDeviceManager::createDefaultAudioDevice(const EDataFlow& dataFlow) @@ -186,6 +237,13 @@ namespace jni std::string name = WideStrToStr(pv.pwszVal); AudioDevicePtr device = std::make_shared(name, id); + if (dataFlow == eCapture) { + device->directionType = AudioDeviceDirectionType::adtCapture; + } else { + device->directionType = AudioDeviceDirectionType::adtRender; + } + + fillAdditionalTypes(device); PropVariantClear(&pv); @@ -223,6 +281,8 @@ namespace jni AudioDevicePtr device = std::make_shared(name, id); + fillAdditionalTypes(device); + if (dataFlow != nullptr) { *dataFlow = flow; } @@ -239,10 +299,14 @@ namespace jni } if (dataFlow == eCapture) { + device->directionType = jni::avdev::AudioDeviceDirectionType::adtCapture; + captureDevices.insertDevice(device); return true; } else if (dataFlow == eRender) { + device->directionType = jni::avdev::AudioDeviceDirectionType::adtRender; + playbackDevices.insertDevice(device); return true; } @@ -250,7 +314,7 @@ namespace jni return false; } - void WindowsAudioDeviceManager::removeAudioDevice(DeviceList & devices, std::string id) + void WindowsAudioDeviceManager::removeAudioDevice(DeviceList & devices, std::string id, EDataFlow dataFlow) { auto predicate = [id](const AudioDevicePtr & dev) { return id == dev->getDescriptor(); @@ -259,6 +323,12 @@ namespace jni AudioDevicePtr removed = devices.removeDevice(predicate); if (removed) { + if (dataFlow == eCapture) { + removed->directionType = AudioDeviceDirectionType::adtCapture; + } else { + removed->directionType = AudioDeviceDirectionType::adtRender; + } + notifyDeviceDisconnected(removed); } } @@ -379,5 +449,40 @@ namespace jni { return S_OK; } + + DeviceFormFactor WindowsAudioDeviceManager::getActualFormFactor(EndpointFormFactor formFactor) { + switch (formFactor) { + case RemoteNetworkDevice: + case UnknownDigitalPassthrough: + case EndpointFormFactor_enum_count: + case UnknownFormFactor: + case LineLevel: + case SPDIF: + return DeviceFormFactor::ffUnknown; + case Speakers: + return DeviceFormFactor::ffSpeaker; + case Headphones: + return DeviceFormFactor::ffHeadphone; + case Headset: + case Handset: + return DeviceFormFactor::ffHeadset; + case Microphone: + return DeviceFormFactor::ffMicrophone; + case DigitalAudioDisplayDevice: // for audio devices this means hdmi + return DeviceFormFactor::ffUnknown; + default: + return DeviceFormFactor::ffUnknown; + } + } + + DeviceTransport WindowsAudioDeviceManager::getActualTransport(EndpointFormFactor formFactor) { + // for Windows, only HDMI can be reliably determined + switch (formFactor) { + case DigitalAudioDisplayDevice: + return DeviceTransport::trHdmi; + default: + return DeviceTransport::trUnknown; + } + } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/media/video/windows/WindowsVideoDeviceManager.cpp b/webrtc-jni/src/main/cpp/src/media/video/windows/WindowsVideoDeviceManager.cpp index ae1e9817..936c59cb 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/windows/WindowsVideoDeviceManager.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/windows/WindowsVideoDeviceManager.cpp @@ -87,6 +87,8 @@ namespace jni uint32_t deviceCount = info->NumberOfDevices(); if (deviceCount > 0) { + // fix duplicated item when device reconect to port. The comparator does not always work correctly. + captureDevices.clearDevices(); const uint32_t size = webrtc::kVideoCaptureDeviceNameLength; for (uint32_t i = 0; i < deviceCount; ++i) { diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/AudioDeviceDirectionType.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/AudioDeviceDirectionType.java new file mode 100644 index 00000000..1338624c --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/AudioDeviceDirectionType.java @@ -0,0 +1,7 @@ +package dev.onvoid.webrtc.media; + +public enum AudioDeviceDirectionType { + UNKNOWN, + CAPTURE, + RENDER +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/Device.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/Device.java index 9b0c87f1..e31c0edf 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/Device.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/Device.java @@ -24,6 +24,9 @@ public abstract class Device { private final String name; + private DeviceTransport deviceTransport = DeviceTransport.UNKNOWN; + + private DeviceFormFactor deviceFormFactor = DeviceFormFactor.UNKNOWN; protected Device(String name, String descriptor) { this.name = name; @@ -38,6 +41,14 @@ public String getName() { return name; } + public DeviceTransport getDeviceTransport() { + return deviceTransport; + } + + public DeviceFormFactor getDeviceFormFactor() { + return deviceFormFactor; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -60,7 +71,7 @@ public int hashCode() { @Override public String toString() { - return String.format("%s [name=%s, descriptor=%s]", - Device.class.getSimpleName(), name, descriptor); + return String.format("%s [name=%s, descriptor=%s, transport=%s, formFactor=%s]", + Device.class.getSimpleName(), name, descriptor, deviceTransport, deviceFormFactor); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceFormFactor.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceFormFactor.java new file mode 100644 index 00000000..5783b7c6 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceFormFactor.java @@ -0,0 +1,9 @@ +package dev.onvoid.webrtc.media; + +public enum DeviceFormFactor { + UNKNOWN, + SPEAKER, + MICROPHONE, + HEADSET, + HEADPHONE; +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceTransport.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceTransport.java new file mode 100644 index 00000000..fa31c86b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/DeviceTransport.java @@ -0,0 +1,8 @@ +package dev.onvoid.webrtc.media; + +public enum DeviceTransport { + UNKNOWN, + HDMI, + USB, + WIRELESS; +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDevice.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDevice.java index 23b19c7e..c2a53357 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDevice.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioDevice.java @@ -16,12 +16,19 @@ package dev.onvoid.webrtc.media.audio; +import dev.onvoid.webrtc.media.AudioDeviceDirectionType; import dev.onvoid.webrtc.media.Device; public class AudioDevice extends Device { + private AudioDeviceDirectionType directionType = AudioDeviceDirectionType.UNKNOWN; + + protected AudioDevice(String name, String descriptor) { super(name, descriptor); } + public AudioDeviceDirectionType getDirectionType() { + return directionType; + } }