Autoplay muted videos stop playback of any streaming app in the backg…

…round

https://bugs.webkit.org/show_bug.cgi?id=163993
<rdar://problem/29020431>

Reviewed by Eric Carlson.

Source/WebCore:

Added test in TestWebKitAPI, WebKit1.AudioSessionCategoryIOS.

Previously, we would set the audio session category to "playback" if there was a media-
element-type media session, and if there was a session capable of playing audio. But because
this was an "or" operation, we would incorrectly set the category to "playback" if there was
a video element incapable of rendering audio (due to being muted, without an audio track,
etc.), and also a session capable of producing audio, such as WebAudio.

With this change, this turns into an "and" operation; there must be a media element capable
of rendering audio in order to switch the audio session category to "playback".

Additionally, we no longer cache the value of "canProduceAudio()"; it's queried directly
whenever updateSessionState() is called.

* Modules/webaudio/AudioContext.cpp:
(WebCore::AudioContext::constructCommon):
* Modules/webaudio/AudioContext.h:
* html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::insertedInto):
(WebCore::HTMLMediaElement::loadResource):
(WebCore::HTMLMediaElement::setMuted):
(WebCore::HTMLMediaElement::mediaPlayerCurrentPlaybackTargetIsWirelessChanged):
(WebCore::HTMLMediaElement::mediaPlayerCharacteristicChanged):
(WebCore::HTMLMediaElement::clearMediaPlayer):
(WebCore::HTMLMediaElement::mediaPlayerCurrentPlaybackTargetIsWirelessChanged):
(WebCore::HTMLMediaElement::presentationType):
(WebCore::HTMLMediaElement::characteristics):
(WebCore::HTMLMediaElement::canProduceAudio):
* html/HTMLMediaElement.h:
* platform/audio/PlatformMediaSession.cpp:
(WebCore::PlatformMediaSession::activeAudioSessionRequired):
(WebCore::PlatformMediaSession::canProduceAudio):
(WebCore::PlatformMediaSession::canProduceAudioChanged):
(WebCore::PlatformMediaSession::setCanProduceAudio): Deleted.
* platform/audio/PlatformMediaSession.h:
(WebCore::PlatformMediaSessionClient::canProduceAudio):
(WebCore::PlatformMediaSession::canProduceAudio): Deleted.
* platform/audio/cocoa/MediaSessionManagerCocoa.cpp:
(PlatformMediaSessionManager::updateSessionState):

Tools:

* TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
* TestWebKitAPI/Tests/WebKit/ios/AudioSessionCategoryIOS.mm:
(TestWebKitAPI::TEST):
* TestWebKitAPI/Tests/WebKit/ios/video-with-muted-audio-and-webaudio.html: Added.


Canonical link: https://commits.webkit.org/184516@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@211240 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog

@@ -1,3 +1,51 @@
+2017-01-26  Jer Noble  <jer.noble@apple.com>
+
+        Autoplay muted videos stop playback of any streaming app in the background
+        https://bugs.webkit.org/show_bug.cgi?id=163993
+        <rdar://problem/29020431>
+
+        Reviewed by Eric Carlson.
+
+        Added test in TestWebKitAPI, WebKit1.AudioSessionCategoryIOS.
+
+        Previously, we would set the audio session category to "playback" if there was a media-
+        element-type media session, and if there was a session capable of playing audio. But because
+        this was an "or" operation, we would incorrectly set the category to "playback" if there was
+        a video element incapable of rendering audio (due to being muted, without an audio track,
+        etc.), and also a session capable of producing audio, such as WebAudio.
+
+        With this change, this turns into an "and" operation; there must be a media element capable
+        of rendering audio in order to switch the audio session category to "playback".
+
+        Additionally, we no longer cache the value of "canProduceAudio()"; it's queried directly
+        whenever updateSessionState() is called.
+
+        * Modules/webaudio/AudioContext.cpp:
+        (WebCore::AudioContext::constructCommon):
+        * Modules/webaudio/AudioContext.h:
+        * html/HTMLMediaElement.cpp:
+        (WebCore::HTMLMediaElement::insertedInto):
+        (WebCore::HTMLMediaElement::loadResource):
+        (WebCore::HTMLMediaElement::setMuted):
+        (WebCore::HTMLMediaElement::mediaPlayerCurrentPlaybackTargetIsWirelessChanged):
+        (WebCore::HTMLMediaElement::mediaPlayerCharacteristicChanged):
+        (WebCore::HTMLMediaElement::clearMediaPlayer):
+        (WebCore::HTMLMediaElement::mediaPlayerCurrentPlaybackTargetIsWirelessChanged):
+        (WebCore::HTMLMediaElement::presentationType):
+        (WebCore::HTMLMediaElement::characteristics):
+        (WebCore::HTMLMediaElement::canProduceAudio):
+        * html/HTMLMediaElement.h:
+        * platform/audio/PlatformMediaSession.cpp:
+        (WebCore::PlatformMediaSession::activeAudioSessionRequired):
+        (WebCore::PlatformMediaSession::canProduceAudio):
+        (WebCore::PlatformMediaSession::canProduceAudioChanged):
+        (WebCore::PlatformMediaSession::setCanProduceAudio): Deleted.
+        * platform/audio/PlatformMediaSession.h:
+        (WebCore::PlatformMediaSessionClient::canProduceAudio):
+        (WebCore::PlatformMediaSession::canProduceAudio): Deleted.
+        * platform/audio/cocoa/MediaSessionManagerCocoa.cpp:
+        (PlatformMediaSessionManager::updateSessionState):
+
 2017-01-26  Filip Pizlo  <fpizlo@apple.com>
 
         EventTarget should visit the JSEventListeners using visitAdditionalChildren

Source/WebCore/Modules/webaudio/AudioContext.cpp

@@ -181,8 +181,6 @@ void AudioContext::constructCommon()
 #if PLATFORM(COCOA)
     addBehaviorRestriction(RequirePageConsentForAudioStartRestriction);
 #endif
-
-    m_mediaSession->setCanProduceAudio(true);
 }
 
 AudioContext::~AudioContext()

Source/WebCore/Modules/webaudio/AudioContext.h

@@ -317,6 +317,7 @@ class AudioContext : public ActiveDOMObject, public ThreadSafeRefCounted<AudioCo
     bool supportsSeeking() const override { return false; }
     bool shouldOverrideBackgroundPlaybackRestriction(PlatformMediaSession::InterruptionType) const override { return false; }
     String sourceApplicationIdentifier() const override;
+    bool canProduceAudio() const final { return true; }
 
     // EventTarget
     void refEventTarget() override { ref(); }

Source/WebCore/html/HTMLMediaElement.cpp

@@ -832,6 +832,7 @@ Node::InsertionNotificationRequest HTMLMediaElement::insertedInto(ContainerNode&
     if (!m_explicitlyMuted) {
         m_explicitlyMuted = true;
         m_muted = hasAttributeWithoutSynchronization(mutedAttr);
+        m_mediaSession->canProduceAudioChanged();
     }
 
     return InsertionShouldCallFinishedInsertingSubtree;
@@ -1485,6 +1486,7 @@ void HTMLMediaElement::loadResource(const URL& initialURL, ContentType& contentT
     if (!m_explicitlyMuted) {
         m_explicitlyMuted = true;
         m_muted = hasAttributeWithoutSynchronization(mutedAttr);
+        m_mediaSession->canProduceAudioChanged();
     }
 
     updateVolume();
@@ -3330,6 +3332,7 @@ void HTMLMediaElement::setMuted(bool muted)
 #if ENABLE(WIRELESS_PLAYBACK_TARGET)
         updateMediaState(UpdateState::Asynchronously);
 #endif
+        m_mediaSession->canProduceAudioChanged();
     }
 
     scheduleUpdatePlaybackControlsManager();
@@ -4663,8 +4666,6 @@ void HTMLMediaElement::mediaPlayerCharacteristicChanged(MediaPlayer*)
     if (!paused() && !m_mediaSession->playbackPermitted(*this))
         pauseInternal();
 
-    m_mediaSession->setCanProduceAudio(m_player && m_readyState >= HAVE_METADATA && hasAudio());
-
 #if ENABLE(MEDIA_SESSION)
     document().updateIsPlayingMedia(m_elementID);
 #else
@@ -5082,8 +5083,8 @@ void HTMLMediaElement::clearMediaPlayer(DelayedActionType flags)
         configureTextTrackDisplay();
 #endif
 
-    m_mediaSession->setCanProduceAudio(false);
     m_mediaSession->clientCharacteristicsChanged();
+    m_mediaSession->canProduceAudioChanged();
 
     updateSleepDisabling();
 }
@@ -5274,8 +5275,7 @@ void HTMLMediaElement::mediaPlayerCurrentPlaybackTargetIsWirelessChanged(MediaPl
     configureMediaControls();
     scheduleEvent(eventNames().webkitcurrentplaybacktargetiswirelesschangedEvent);
     m_mediaSession->isPlayingToWirelessPlaybackTargetChanged(m_isPlayingToWirelessTarget);
-    if (m_isPlayingToWirelessTarget)
-        m_mediaSession->setCanProduceAudio(true);
+    m_mediaSession->canProduceAudioChanged();
     updateMediaState(UpdateState::Asynchronously);
 }
 
@@ -6819,7 +6819,7 @@ PlatformMediaSession::MediaType HTMLMediaElement::mediaType() const
 PlatformMediaSession::MediaType HTMLMediaElement::presentationType() const
 {
     if (hasTagName(HTMLNames::videoTag))
-        return PlatformMediaSession::VideoAudio;
+        return muted() ? PlatformMediaSession::Video : PlatformMediaSession::VideoAudio;
 
     return PlatformMediaSession::Audio;
 }
@@ -6851,6 +6851,21 @@ PlatformMediaSession::CharacteristicsFlags HTMLMediaElement::characteristics() c
     return state;
 }
 
+bool HTMLMediaElement::canProduceAudio() const
+{
+#if ENABLE(WIRELESS_PLAYBACK_TARGET)
+    // Because the remote target could unmute playback without notifying us, we must assume
+    // that we may be playing audio.
+    if (m_isPlayingToWirelessTarget)
+        return true;
+#endif
+
+    if (muted())
+        return false;
+
+    return m_player && m_readyState >= HAVE_METADATA && hasAudio();
+}
+
 #if ENABLE(MEDIA_SOURCE)
 size_t HTMLMediaElement::maximumSourceBufferSize(const SourceBuffer& buffer) const
 {

Source/WebCore/html/HTMLMediaElement.h

@@ -776,6 +776,7 @@ class HTMLMediaElement
     bool supportsSeeking() const override;
     bool shouldOverrideBackgroundPlaybackRestriction(PlatformMediaSession::InterruptionType) const override;
     bool shouldOverrideBackgroundLoadingRestriction() const override;
+    bool canProduceAudio() const final;
 
     void pageMutedStateDidChange() override;
 

Source/WebCore/platform/audio/PlatformMediaSession.cpp

@@ -329,15 +329,16 @@ bool PlatformMediaSession::activeAudioSessionRequired()
         return false;
     if (state() != PlatformMediaSession::State::Playing)
         return false;
-    return m_canProduceAudio;
+    return canProduceAudio();
 }
 
-void PlatformMediaSession::setCanProduceAudio(bool canProduceAudio)
+bool PlatformMediaSession::canProduceAudio() const
 {
-    if (m_canProduceAudio == canProduceAudio)
-        return;
-    m_canProduceAudio = canProduceAudio;
+    return m_client.canProduceAudio();
+}
 
+void PlatformMediaSession::canProduceAudioChanged()
+{
     PlatformMediaSessionManager::sharedManager().sessionCanProduceAudioChanged(*this);
 }
 

Source/WebCore/platform/audio/PlatformMediaSession.h

@@ -162,8 +162,8 @@ class PlatformMediaSession
 #endif
 
     bool activeAudioSessionRequired();
-    bool canProduceAudio() const { return m_canProduceAudio; }
-    void setCanProduceAudio(bool);
+    bool canProduceAudio() const;
+    void canProduceAudioChanged();
 
     void scheduleClientDataBufferingCheck();
     virtual void resetPlaybackSessionState() { }
@@ -186,7 +186,6 @@ class PlatformMediaSession
     int m_interruptionCount { 0 };
     bool m_notifyingClient;
     bool m_isPlayingToWirelessPlaybackTarget { false };
-    bool m_canProduceAudio { false };
 
     friend class PlatformMediaSessionManager;
 };
@@ -217,6 +216,7 @@ class PlatformMediaSessionClient {
 
     virtual void setShouldBufferData(bool) { }
     virtual bool elementIsHidden() const { return false; }
+    virtual bool canProduceAudio() const { return false; }
 
     virtual bool shouldOverrideBackgroundPlaybackRestriction(PlatformMediaSession::InterruptionType) const = 0;
     virtual bool shouldOverrideBackgroundLoadingRestriction() const { return false; }

Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.cpp

@@ -31,6 +31,7 @@
 #include "AudioSession.h"
 #include "Logging.h"
 #include "Settings.h"
+#include <wtf/Function.h>
 
 using namespace WebCore;
 
@@ -60,13 +61,20 @@ void PlatformMediaSessionManager::updateSessionState()
     if (!Settings::shouldManageAudioSessionCategory())
         return;
 
-    if (has(PlatformMediaSession::VideoAudio) || has(PlatformMediaSession::Audio)) {
-        if (canProduceAudio())
-            AudioSession::sharedSession().setCategory(AudioSession::MediaPlayback);
-        else
-            AudioSession::sharedSession().setCategory(AudioSession::AmbientSound);
-    } else if (has(PlatformMediaSession::WebAudio))
+    bool hasAudioMediaType = false;
+    bool hasAudibleAudioOrVideoMediaType = anyOfSessions([this, hasAudioMediaType] (PlatformMediaSession& session, size_t) mutable {
+        auto type = session.mediaType();
+        if (type == PlatformMediaSession::VideoAudio || type == PlatformMediaSession::Audio || type == PlatformMediaSession::WebAudio)
+            hasAudioMediaType = true;
+        return (type == PlatformMediaSession::VideoAudio || type == PlatformMediaSession::Audio) && session.canProduceAudio();
+    });
+
+    if (hasAudibleAudioOrVideoMediaType)
+        AudioSession::sharedSession().setCategory(AudioSession::MediaPlayback);
+    else if (hasAudioMediaType)
         AudioSession::sharedSession().setCategory(AudioSession::AmbientSound);
+    else
+        AudioSession::sharedSession().setCategory(AudioSession::None);
 #endif
 }
 

Tools/ChangeLog

@@ -1,3 +1,16 @@
+2017-01-26  Jer Noble  <jer.noble@apple.com>
+
+        Autoplay muted videos stop playback of any streaming app in the background
+        https://bugs.webkit.org/show_bug.cgi?id=163993
+        <rdar://problem/29020431>
+
+        Reviewed by Eric Carlson.
+
+        * TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
+        * TestWebKitAPI/Tests/WebKit/ios/AudioSessionCategoryIOS.mm:
+        (TestWebKitAPI::TEST):
+        * TestWebKitAPI/Tests/WebKit/ios/video-with-muted-audio-and-webaudio.html: Added.
+
 2017-01-26  Ryan Haddad  <ryanhaddad@apple.com>
 
         Disable API test WebKit2.WebsitePoliciesPlayAfterPreventedAutoplay.

Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj

@@ -512,6 +512,7 @@
 		C99B675C1E39721A00FC6C80 /* autoplay-with-controls.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = C99B675A1E3971FC00FC6C80 /* autoplay-with-controls.html */; };
 		C99B675D1E39722000FC6C80 /* js-play-with-controls.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = C99B675B1E3971FC00FC6C80 /* js-play-with-controls.html */; };
 		C99B675F1E39736F00FC6C80 /* no-autoplay-with-controls.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = C99B675E1E39735C00FC6C80 /* no-autoplay-with-controls.html */; };
+		CD321B041E3A85FA00EB21C8 /* video-with-muted-audio-and-webaudio.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = CD321B031E3A84B700EB21C8 /* video-with-muted-audio-and-webaudio.html */; };
 		CD59F53419E9110D00CF1835 /* file-with-mse.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = CD59F53219E910AA00CF1835 /* file-with-mse.html */; };
 		CD59F53519E9110D00CF1835 /* test-mse.mp4 in Copy Resources */ = {isa = PBXBuildFile; fileRef = CD59F53319E910BC00CF1835 /* test-mse.mp4 */; };
 		CD78E11D1DB7EA660014A2DE /* FullscreenDelegate.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD78E11A1DB7EA360014A2DE /* FullscreenDelegate.mm */; };
@@ -613,6 +614,7 @@
 				C99B675F1E39736F00FC6C80 /* no-autoplay-with-controls.html in Copy Resources */,
 				C99B675D1E39722000FC6C80 /* js-play-with-controls.html in Copy Resources */,
 				C99B675C1E39721A00FC6C80 /* autoplay-with-controls.html in Copy Resources */,
+				CD321B041E3A85FA00EB21C8 /* video-with-muted-audio-and-webaudio.html in Copy Resources */,
 				C95984F71E36BCEF002C0D45 /* test-without-audio-track.mp4 in Copy Resources */,
 				C95984F41E36BC6B002C0D45 /* autoplay-check.html in Copy Resources */,
 				C95984F51E36BC6B002C0D45 /* autoplay-no-audio-check.html in Copy Resources */,
@@ -1284,6 +1286,7 @@
 		C99B675B1E3971FC00FC6C80 /* js-play-with-controls.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "js-play-with-controls.html"; sourceTree = "<group>"; };
 		C99B675E1E39735C00FC6C80 /* no-autoplay-with-controls.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "no-autoplay-with-controls.html"; sourceTree = "<group>"; };
 		CD225C071C45A69200140761 /* ParsedContentRange.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParsedContentRange.cpp; sourceTree = "<group>"; };
+		CD321B031E3A84B700EB21C8 /* video-with-muted-audio-and-webaudio.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "video-with-muted-audio-and-webaudio.html"; sourceTree = "<group>"; };
 		CD5393C71757BA9700C07123 /* MD5.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MD5.cpp; sourceTree = "<group>"; };
 		CD5393C91757BAC400C07123 /* SHA1.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SHA1.cpp; sourceTree = "<group>"; };
 		CD5451E919E41F9D0016936F /* CSSParser.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CSSParser.cpp; sourceTree = "<group>"; };
@@ -2235,6 +2238,7 @@
 				CDC8E48B1BC5C96200594FEC /* video-without-audio.html */,
 				CDC8E48C1BC5C96200594FEC /* video-without-audio.mp4 */,
 				CDB411591E09DA8E00EAD352 /* video-with-muted-audio.html */,
+				CD321B031E3A84B700EB21C8 /* video-with-muted-audio-and-webaudio.html */,
 			);
 			name = Resources;
 			sourceTree = "<group>";

Tools/TestWebKitAPI/Tests/WebKit/ios/AudioSessionCategoryIOS.mm

@@ -97,6 +97,14 @@ - (BOOL)webView:(UIWebView *)webView shouldStartLoadWithRequest:(NSURLRequest *)
     Util::run(&didBeginPlaying);
 
     EXPECT_WK_STREQ(getAVAudioSessionCategoryAmbient(), [[getAVAudioSessionClass() sharedInstance] category]);
+
+    didBeginPlaying = false;
+
+    [uiWebView loadRequest:[NSURLRequest requestWithURL:[[NSBundle mainBundle] URLForResource:@"video-with-muted-audio-and-webaudio" withExtension:@"html" subdirectory:@"TestWebKitAPI.resources"]]];
+
+    Util::run(&didBeginPlaying);
+
+    EXPECT_WK_STREQ(getAVAudioSessionCategoryAmbient(), [[getAVAudioSessionClass() sharedInstance] category]);
 }
 
 }

Tools/TestWebKitAPI/Tests/WebKit/ios/video-with-muted-audio-and-webaudio.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <script>
+        function go() {
+            var audioContext = new webkitAudioContext();
+            audioContext.resume();
+
+            var video = document.getElementsByTagName('video')[0];
+            video.play().then(playing).catch(notPlaying);
+        }
+
+        function playing() {
+            try {
+                window.webkit.messageHandlers.testHandler.postMessage('playing');
+            } catch(e) {
+                window.location = 'callback:playing';
+            }
+        }
+
+        function notPlaying() {
+            try {
+                window.webkit.messageHandlers.testHandler.postMessage('not playing');
+            } catch(e) { }
+        }
+        </script>
+    </head>
+    <body onload="go()">
+        <video src="video-with-audio.mp4" muted webkit-playsinline></video>
+    </body>
+</html>