A muted microphone track should get ended if its device disappears

https://bugs.webkit.org/show_bug.cgi?id=255591 rdar://problem/108194510 Reviewed by Eric Carlson. The main change is in BaseAudioSharedUnit::devicesChanged where we no longer exit early if the shared unit is not running. This ensures that muted tracks will be ended if their device is gone. We do a refactoring so that mock device changes kick in the BaseAudioSharedUnit::devicesChanged logic. We also do a small change to MediaStreamTrack::trackEnded to only log MediaStreamTrack capture failure if the track is not already ended. Covered by added test. Drive by fix in LayoutTests/fast/mediastream/microphone-change-while-capturing.html to ensure we use the usb fake device. Rebasing of some tests now that some additional tracks are failing due to device changes. * LayoutTests/fast/mediastream/MediaDevices-addEventListener-expected.txt: * LayoutTests/fast/mediastream/device-change-event-2-expected.txt: * LayoutTests/fast/mediastream/microphone-change-while-capturing-expected.txt: * LayoutTests/fast/mediastream/microphone-change-while-capturing.html: * LayoutTests/fast/mediastream/microphone-change-while-muted-expected.txt: Added. * LayoutTests/fast/mediastream/microphone-change-while-muted.html: Copied from LayoutTests/fast/mediastream/microphone-change-while-capturing.html. * LayoutTests/platform/glib/TestExpectations: * Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp: (WebCore::MediaStreamTrack::trackEnded): * Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp: (WebCore::BaseAudioSharedUnit::BaseAudioSharedUnit): (WebCore::BaseAudioSharedUnit::~BaseAudioSharedUnit): (WebCore::BaseAudioSharedUnit::devicesChanged): * Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h: * Source/WebCore/platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp: (WebCore::CoreAudioCaptureDeviceManager::refreshAudioCaptureDevices): * Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp: (WebCore::CoreAudioCaptureSourceFactory::devicesChanged): Deleted. * Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h: Canonical link: https://commits.webkit.org/263132@main
WebKit · Apr 19, 2023 · aa24abe · aa24abe
1 parent 6e008ff
commit aa24abe
Show file tree

Hide file tree

Showing 13 changed files with 109 additions and 21 deletions.
diff --git a/LayoutTests/fast/mediastream/MediaDevices-addEventListener-expected.txt b/LayoutTests/fast/mediastream/MediaDevices-addEventListener-expected.txt
@@ -1,3 +1,4 @@
+CONSOLE MESSAGE: A MediaStreamTrack ended due to a capture failure
 
 PASS Testing MediaDevices addEventListener/removeEventListener
 PASS Capture 'devicechange' event with addEventListener

diff --git a/LayoutTests/fast/mediastream/device-change-event-2-expected.txt b/LayoutTests/fast/mediastream/device-change-event-2-expected.txt
@@ -1,3 +1,5 @@
+CONSOLE MESSAGE: A MediaStreamTrack ended due to a capture failure
+CONSOLE MESSAGE: A MediaStreamTrack ended due to a capture failure
 
 PASS 'devicechange' event fired when device list changes
 PASS 'devicechange' events fired quickly are coalesced

diff --git a/LayoutTests/fast/mediastream/microphone-change-while-capturing-expected.txt b/LayoutTests/fast/mediastream/microphone-change-while-capturing-expected.txt
@@ -1,3 +1,4 @@
+CONSOLE MESSAGE: A MediaStreamTrack ended due to a capture failure
 
 
 PASS Detection of missing capturing device should trigger capture to fail

diff --git a/LayoutTests/fast/mediastream/microphone-change-while-capturing.html b/LayoutTests/fast/mediastream/microphone-change-while-capturing.html
@@ -16,16 +16,32 @@
         test.add_cleanup(() => { testRunner.resetMockMediaDevices(); });
     }
 
+    async function getDeviceId(label)
+    {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        const devices = await navigator.mediaDevices.enumerateDevices();
+        let deviceId;
+        devices.forEach(device => {
+            if (device.label === "my USB microphone")
+                deviceId = device.deviceId;
+        });
+
+        stream.getAudioTracks()[0].stop();
+
+        return deviceId;
+    }
+
     promise_test(async (test) => {
         await setup(test);
 
         testRunner.addMockMicrophoneDevice("usbmic", "my USB microphone");
-        video.srcObject = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: "usbmic" } });
+        const deviceId = await getDeviceId("my USB microphone");
+        video.srcObject = await navigator.mediaDevices.getUserMedia({ audio: { deviceId } });
         await video.play();
 
         testRunner.removeMockMediaDevice("usbmic");
         return new Promise((resolve, reject) => {
-            video.srcObject.getAudioTracks()[0].onended = resolve();
+            video.srcObject.getAudioTracks()[0].onended = resolve;
             setTimeout(reject, 2000);
         });
     }, "Detection of missing capturing device should trigger capture to fail");
@@ -35,7 +51,8 @@
 
         testRunner.addMockMicrophoneDevice("usbmic1", "my USB microphone");
         testRunner.addMockMicrophoneDevice("usbmic2", "my second USB microphone");
-        video.srcObject = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: "usbmic1" } });
+        const deviceId = await getDeviceId("my USB microphone");
+        video.srcObject = await navigator.mediaDevices.getUserMedia({ audio: { deviceId } });
         await video.play();
         testRunner.removeMockMediaDevice("usbmic2");
         testRunner.addMockMicrophoneDevice("usbmic3", "my third USB microphone");

diff --git a/LayoutTests/fast/mediastream/microphone-change-while-muted-expected.txt b/LayoutTests/fast/mediastream/microphone-change-while-muted-expected.txt
@@ -0,0 +1,5 @@
+CONSOLE MESSAGE: A MediaStreamTrack ended due to a capture failure
+
+
+PASS Detection of missing capturing device should trigger capture to fail even if track is muted
+
diff --git a/LayoutTests/fast/mediastream/microphone-change-while-muted.html b/LayoutTests/fast/mediastream/microphone-change-while-muted.html
@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>Testing change of device while capturing.</title>
+    <script src="../../resources/testharness.js"></script>
+    <script src="../../resources/testharnessreport.js"></script>
+</head>
+<body>
+    <video id="video"></video>
+    <script>
+    let setup = async (test) => {
+        if (!window.testRunner)
+            return Promise.reject("test requires internal API");
+
+        test.add_cleanup(() => { testRunner.resetMockMediaDevices(); });
+    }
+
+    async function getDeviceId(label)
+    {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        const devices = await navigator.mediaDevices.enumerateDevices();
+        let deviceId;
+        devices.forEach(device => {
+            if (device.label === "my USB microphone")
+                deviceId = device.deviceId;
+        });
+
+        stream.getAudioTracks()[0].stop();
+
+        return deviceId;
+    }
+
+    promise_test(async (test) => {
+        await setup(test);
+
+        testRunner.addMockMicrophoneDevice("usbmic", "my USB microphone");
+
+        const deviceId = await getDeviceId("my USB microphone");
+        video.srcObject = await navigator.mediaDevices.getUserMedia({ audio: { deviceId } });
+        await video.play();
+
+        if (window.internals)
+            internals.setPageMuted("capturedevices");
+
+        await new Promise(resolve => video.srcObject.getAudioTracks()[0].onmute = resolve);
+
+        testRunner.removeMockMediaDevice("usbmic");
+        return new Promise((resolve, reject) => {
+            video.srcObject.getAudioTracks()[0].onended = resolve;
+            setTimeout(() => reject("track did not end"), 2000);
+        });
+    }, "Detection of missing capturing device should trigger capture to fail even if track is muted");
+    </script>
+</body>
+</html>
diff --git a/LayoutTests/platform/glib/TestExpectations b/LayoutTests/platform/glib/TestExpectations
@@ -1683,7 +1683,11 @@ webkit.org/b/187603 fast/mediastream/media-stream-track-source-failure.html [ Ti
 # This ends up calling WKPageTriggerMockMicrophoneConfigurationChange which is specific to GPUProcess.
 fast/mediastream/mediastreamtrack-configurationchange.html [ Skip ]
 
-fast/mediastream/MediaDevices-addEventListener.html [ DumpJSConsoleLogInStdErr ]
+# Tests should be rebased, mock infra might need some updates.
+fast/mediastream/MediaDevices-addEventListener.html [ Failure ]
+fast/mediastream/microphone-change-while-capturing.html [ Failure ]
+fast/mediastream/microphone-change-while-muted.html [ Failure ]
+fast/mediastream/device-change-event-2.html [ Failure ]
 
 webkit.org/b/194611 http/wpt/webrtc/getUserMedia-processSwapping.html [ Failure ]
 

diff --git a/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp b/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp
@@ -505,7 +505,7 @@ void MediaStreamTrack::trackEnded(MediaStreamTrackPrivate&)
 
     ALWAYS_LOG(LOGIDENTIFIER);
 
-    if (m_isCaptureTrack && m_private->source().captureDidFail())
+    if (m_isCaptureTrack && m_private->source().captureDidFail() && m_readyState != State::Ended)
         scriptExecutionContext()->addConsoleMessage(MessageSource::JS, MessageLevel::Error, "A MediaStreamTrack ended due to a capture failure"_s);
 
     // http://w3c.github.io/mediacapture-main/#life-cycle

diff --git a/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp b/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.cpp
@@ -29,6 +29,7 @@
 #if ENABLE(MEDIA_STREAM)
 
 #include "AudioSession.h"
+#include "CaptureDeviceManager.h"
 #include "CoreAudioCaptureSource.h"
 #include "DeprecatedGlobalSettings.h"
 #include "Logging.h"
@@ -40,6 +41,12 @@ namespace WebCore {
 BaseAudioSharedUnit::BaseAudioSharedUnit()
     : m_sampleRate(AudioSession::sharedSession().sampleRate())
 {
+    RealtimeMediaSourceCenter::singleton().addDevicesChangedObserver(*this);
+}
+
+BaseAudioSharedUnit::~BaseAudioSharedUnit()
+{
+    RealtimeMediaSourceCenter::singleton().removeDevicesChangedObserver(*this);
 }
 
 void BaseAudioSharedUnit::addClient(CoreAudioCaptureSource& client)
@@ -150,12 +157,13 @@ void BaseAudioSharedUnit::setCaptureDevice(String&& persistentID, uint32_t captu
         captureDeviceChanged();
 }
 
-void BaseAudioSharedUnit::devicesChanged(const Vector<CaptureDevice>& devices)
+void BaseAudioSharedUnit::devicesChanged()
 {
-    if (!m_producingCount)
+    auto devices = RealtimeMediaSourceCenter::singleton().audioCaptureFactory().audioCaptureDeviceManager().captureDevices();
+    auto persistentID = this->persistentID();
+    if (persistentID.isEmpty())
         return;
 
-    auto persistentID = this->persistentID();
     if (WTF::anyOf(devices, [&persistentID] (auto& device) { return persistentID == device.persistentId(); })) {
         validateOutputDevice(m_outputDeviceID);
         return;

diff --git a/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h b/Source/WebCore/platform/mediastream/mac/BaseAudioSharedUnit.h
@@ -28,6 +28,7 @@
 #if ENABLE(MEDIA_STREAM)
 
 #include "RealtimeMediaSourceCapabilities.h"
+#include "RealtimeMediaSourceCenter.h"
 #include <wtf/Function.h>
 #include <wtf/HashSet.h>
 #include <wtf/Lock.h>
@@ -42,10 +43,10 @@ class CaptureDevice;
 class CoreAudioCaptureSource;
 class PlatformAudioData;
 
-class BaseAudioSharedUnit : public CanMakeWeakPtr<BaseAudioSharedUnit, WeakPtrFactoryInitialization::Eager> {
+class BaseAudioSharedUnit : public RealtimeMediaSourceCenter::Observer {
 public:
     BaseAudioSharedUnit();
-    virtual ~BaseAudioSharedUnit() = default;
+    virtual ~BaseAudioSharedUnit();
 
     void startProducingData();
     void stopProducingData();
@@ -79,7 +80,6 @@ class BaseAudioSharedUnit : public CanMakeWeakPtr<BaseAudioSharedUnit, WeakPtrFa
     virtual CapabilityValueOrRange sampleRateCapacities() const = 0;
     virtual int actualSampleRate() const { return sampleRate(); }
 
-    void devicesChanged(const Vector<CaptureDevice>&);
     void whenAudioCaptureUnitIsNotRunning(Function<void()>&&);
     bool isRenderingAudio() const { return m_isRenderingAudio; }
     bool hasClients() const { return !m_clients.isEmpty(); }
@@ -119,6 +119,9 @@ class BaseAudioSharedUnit : public CanMakeWeakPtr<BaseAudioSharedUnit, WeakPtrFa
 private:
     OSStatus startUnit();
 
+    // RealtimeMediaSourceCenter::Observer
+    void devicesChanged() final;
+
     bool m_enableEchoCancellation { true };
     double m_volume { 1 };
     int m_sampleRate;

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp b/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureDeviceManager.cpp
@@ -375,10 +375,8 @@ void CoreAudioCaptureDeviceManager::refreshAudioCaptureDevices(NotifyIfDevicesHa
             m_speakerDevices.append(device);
     }
 
-    if (notify == NotifyIfDevicesHaveChanged::Notify) {
+    if (notify == NotifyIfDevicesHaveChanged::Notify)
         deviceChanged();
-        CoreAudioCaptureSourceFactory::singleton().devicesChanged(m_captureDevices);
-    }
 }
 
 } // namespace WebCore

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp b/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.cpp
@@ -176,11 +176,6 @@ const Vector<CaptureDevice>& CoreAudioCaptureSourceFactory::speakerDevices() con
 #endif
 }
 
-void CoreAudioCaptureSourceFactory::devicesChanged(const Vector<CaptureDevice>& devices)
-{
-    CoreAudioSharedUnit::unit().devicesChanged(devices);
-}
-
 void CoreAudioCaptureSourceFactory::registerSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer& producer)
 {
     CoreAudioSharedUnit::unit().registerSpeakerSamplesProducer(producer);

diff --git a/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h b/Source/WebCore/platform/mediastream/mac/CoreAudioCaptureSource.h
@@ -131,8 +131,6 @@ class CoreAudioCaptureSourceFactory : public AudioCaptureFactory, public AudioSe
 
     void scheduleReconfiguration();
 
-    void devicesChanged(const Vector<CaptureDevice>&);
-
     WEBCORE_EXPORT void registerSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer&);
     WEBCORE_EXPORT void unregisterSpeakerSamplesProducer(CoreAudioSpeakerSamplesProducer&);
     WEBCORE_EXPORT bool isAudioCaptureUnitRunning();