Refactor MediaSessionManagerCocoa::nowPlayingEligibleSession to allow…

… non HTMLMediaElement eligible sessions

https://bugs.webkit.org/show_bug.cgi?id=270090
rdar://123632916

Reviewed by Jean-Yves Avenard.

We want to allow AudioContext to be used as media playback with AudioSession API.
For that, it needs to be on par with HTMLMediaElement, and for instance trigger NowPlayingInfo updates.
This patch is refactoring code to untie a bit HTMLMediaElement and NowPlayingInfo computation.
It should not have any impact on behavior since this patch is not adding code to actually allow
AudioContext to be handled like HTMLMediaElement (for instance pay/pause from the control center).

We introduce MediaSession::updateNowPlayingInfo and NavigatorMediaSession::mediaSessionIfExists to
allow updating NowPlayingInfo according MediaSession from either HTMLMediaElement or other media producers.

We introduce getters for nowPlayingInfo and isNowPlayingEligible on PlatformMediaSession which are being forwarded to each client.
We update HTMLMediaElement/MediaElementSession implementation accordingly.

We introduce PlatformMediaSessionManager::bestEligibleSessionForRemoteControls to filter sessions according presentation types.
We then get two lists of media sessions, one of web audio and one of media (HTMLMediaElement basically).
If the HTMLMediaElement list is empty, we look at WebAudio sessions, otherwise we look at HTMLMediaElement sessions.

We introduce selectBestMediaSession so that sorting is done by each session subtype.

This allows to fix the layering violation from MediaSessionManagerCocoa::nowPlayingEligibleSession.:

* Source/WebCore/Modules/mediasession/MediaSession.cpp:
(WebCore::MediaSession::updateNowPlayingInfo):
* Source/WebCore/Modules/mediasession/MediaSession.h:
* Source/WebCore/html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::bestMediaElementForRemoteControls):
(WebCore::HTMLMediaElement::nowPlayingInfo const):
(WebCore::HTMLMediaElement::selectBestMediaSession):
* Source/WebCore/html/HTMLMediaElement.h:
* Source/WebCore/html/MediaElementSession.cpp:
(WebCore::MediaElementSession::hasNowPlayingInfo const):
(WebCore::MediaElementSession::computeNowPlayingInfo const):
(WebCore::MediaElementSession::nowPlayingInfo const): Deleted.
* Source/WebCore/html/MediaElementSession.h:
* Source/WebCore/platform/audio/PlatformMediaSession.cpp:
(WebCore::PlatformMediaSession::nowPlayingInfo const):
(WebCore::PlatformMediaSession::isNowPlayingEligible const):
(WebCore::PlatformMediaSession::selectBestMediaSession):
(WebCore::PlatformMediaSessionClient::nowPlayingInfo const):
* Source/WebCore/platform/audio/PlatformMediaSession.h:
(WebCore::PlatformMediaSessionClient::isNowPlayingEligible const):
(WebCore::PlatformMediaSessionClient::selectBestMediaSession):
* Source/WebCore/platform/audio/PlatformMediaSessionManager.cpp:
(WebCore::PlatformMediaSessionManager::bestEligibleSessionForRemoteControls):
* Source/WebCore/platform/audio/PlatformMediaSessionManager.h:
* Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.h:
* Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.mm:
(WebCore::MediaSessionManagerCocoa::nowPlayingEligibleSession):
(WebCore::MediaSessionManagerCocoa::updateNowPlayingInfo):

Canonical link: https://commits.webkit.org/275460@main

Source/WebCore/Modules/mediasession/MediaSession.cpp

@@ -434,6 +434,24 @@ void MediaSession::willPausePlayback()
     notifyPositionStateObservers();
 }
 
+void MediaSession::updateNowPlayingInfo(NowPlayingInfo& info)
+{
+    if (auto positionState = this->positionState()) {
+        info.duration = positionState->duration;
+        info.rate = positionState->playbackRate;
+    }
+    if (auto currentPosition = this->currentPosition())
+        info.currentTime = *currentPosition;
+
+    if (m_metadata && m_metadata->artworkImage()) {
+        ASSERT(m_metadata->artworkImage()->data(), "An image must always have associated data");
+        info.artwork = { { m_metadata->artworkSrc(), m_metadata->artworkImage()->mimeType(), m_metadata->artworkImage() } };
+        info.title = m_metadata->title();
+        info.artist = m_metadata->artist();
+        info.album = m_metadata->album();
+    }
+}
+
 #if ENABLE(MEDIA_SESSION_COORDINATOR)
 void MediaSession::notifyReadyStateObservers()
 {

Source/WebCore/Modules/mediasession/MediaSession.h

@@ -49,6 +49,7 @@ class MediaSessionCoordinator;
 class MediaSessionCoordinatorPrivate;
 class Navigator;
 template<typename> class DOMPromiseDeferred;
+struct NowPlayingInfo;
 
 class MediaSession : public RefCounted<MediaSession>, public ActiveDOMObject, public CanMakeWeakPtr<MediaSession> {
     WTF_MAKE_FAST_ALLOCATED;
@@ -120,6 +121,8 @@ class MediaSession : public RefCounted<MediaSession>, public ActiveDOMObject, pu
 
     RefPtr<HTMLMediaElement> activeMediaElement() const;
 
+    void updateNowPlayingInfo(NowPlayingInfo&);
+
 private:
     explicit MediaSession(Navigator&);
 

Source/WebCore/html/HTMLMediaElement.cpp

@@ -661,18 +661,33 @@ std::optional<MediaPlayerIdentifier> HTMLMediaElement::playerIdentifier() const
 
 RefPtr<HTMLMediaElement> HTMLMediaElement::bestMediaElementForRemoteControls(MediaElementSession::PlaybackControlsPurpose purpose, const Document* document)
 {
-    Vector<MediaElementSessionInfo> candidateSessions;
-    bool atLeastOneNonCandidateMayBeConfusedForMainContent = false;
-    PlatformMediaSessionManager::sharedManager().forEachMatchingSession([document](auto& session) {
+    auto selectedSession = PlatformMediaSessionManager::sharedManager().bestEligibleSessionForRemoteControls([&document] (auto& session) {
         auto* mediaElementSession = dynamicDowncast<MediaElementSession>(session);
         return mediaElementSession && (!document || &mediaElementSession->element().document() == document);
-    }, [&](auto& session) {
-        auto mediaElementSessionInfo = mediaElementSessionInfoForSession(downcast<MediaElementSession>(session), purpose);
+    }, purpose);
+
+    return selectedSession ? RefPtr { &downcast<MediaElementSession>(selectedSession.get())->element() } : nullptr;
+}
+
+std::optional<NowPlayingInfo> HTMLMediaElement::nowPlayingInfo() const
+{
+    return m_mediaSession->computeNowPlayingInfo();
+}
+
+WeakPtr<PlatformMediaSession> HTMLMediaElement::selectBestMediaSession(const Vector<WeakPtr<PlatformMediaSession>>& sessions, PlatformMediaSession::PlaybackControlsPurpose purpose)
+{
+    if (!sessions.size())
+        return nullptr;
+
+    Vector<MediaElementSessionInfo> candidateSessions;
+    bool atLeastOneNonCandidateMayBeConfusedForMainContent = false;
+    for (auto& session : sessions) {
+        auto mediaElementSessionInfo = mediaElementSessionInfoForSession(*downcast<MediaElementSession>(session.get()), purpose);
         if (mediaElementSessionInfo.canShowControlsManager)
             candidateSessions.append(mediaElementSessionInfo);
         else if (mediaSessionMayBeConfusedWithMainContent(mediaElementSessionInfo, purpose))
             atLeastOneNonCandidateMayBeConfusedForMainContent = true;
-    });
+    }
 
     if (!candidateSessions.size())
         return nullptr;
@@ -682,7 +697,7 @@ RefPtr<HTMLMediaElement> HTMLMediaElement::bestMediaElementForRemoteControls(Med
     if (!strongestSessionCandidate.isVisibleInViewportOrFullscreen && !strongestSessionCandidate.isPlayingAudio && atLeastOneNonCandidateMayBeConfusedForMainContent)
         return nullptr;
 
-    return &strongestSessionCandidate.session->element();
+    return strongestSessionCandidate.session.get();
 }
 
 void HTMLMediaElement::registerWithDocument(Document& document)

Source/WebCore/html/HTMLMediaElement.h

@@ -990,6 +990,9 @@ class HTMLMediaElement
     bool hasMediaStreamSource() const final;
     void processIsSuspendedChanged() final;
     bool shouldOverridePauseDuringRouteChange() const final;
+    bool isNowPlayingEligible() const final { return m_mediaSession->hasNowPlayingInfo(); }
+    std::optional<NowPlayingInfo> nowPlayingInfo() const final;
+    WeakPtr<PlatformMediaSession> selectBestMediaSession(const Vector<WeakPtr<PlatformMediaSession>>&, PlatformMediaSession::PlaybackControlsPurpose) final;
 
     void pageMutedStateDidChange() override;
 

Source/WebCore/html/MediaElementSession.cpp

@@ -1236,18 +1236,26 @@ void MediaElementSession::didReceiveRemoteControlCommand(RemoteControlCommandTyp
 }
 #endif
 
-std::optional<NowPlayingInfo> MediaElementSession::nowPlayingInfo() const
+bool MediaElementSession::hasNowPlayingInfo() const
 {
-    RefPtr page = m_element.document().page();
+#if ENABLE(MEDIA_SESSION) && ENABLE(MEDIA_STREAM)
+    if (!canShowControlsManager(MediaElementSession::PlaybackControlsPurpose::NowPlaying))
+        return false;
 
-#if ENABLE(MEDIA_SESSION)
-    auto* session = mediaSession();
+    RefPtr session = mediaSession();
+    if (isDocumentPlayingSeveralMediaStreamsAndCapturing(m_element.document()) && (!session || !session->hasActiveActionHandlers()))
+        return false;
 #endif
 
-#if ENABLE(MEDIA_SESSION) && ENABLE(MEDIA_STREAM)
-    if (isDocumentPlayingSeveralMediaStreamsAndCapturing(m_element.document()) && (!session || !session->hasActiveActionHandlers()))
+    return true;
+}
+
+std::optional<NowPlayingInfo> MediaElementSession::computeNowPlayingInfo() const
+{
+    if (!hasNowPlayingInfo())
         return { };
-#endif
+
+    RefPtr page = m_element.document().page();
 
     bool allowsNowPlayingControlsVisibility = page && !page->isVisibleAndActive();
     bool isPlaying = state() == PlatformMediaSession::State::Playing;
@@ -1267,26 +1275,13 @@ std::optional<NowPlayingInfo> MediaElementSession::nowPlayingInfo() const
         sourceApplicationIdentifier = presentingApplicationBundleIdentifier();
 #endif
 
+    NowPlayingInfo info { m_element.mediaSessionTitle(), emptyString(), emptyString(), sourceApplicationIdentifier, duration, currentTime, rate, supportsSeeking, m_element.mediaUniqueIdentifier(), isPlaying, allowsNowPlayingControlsVisibility, { } };
 #if ENABLE(MEDIA_SESSION)
-    auto positionState = session ? session->positionState() : std::nullopt;
-    auto currentPosition = session ? session->currentPosition() : std::nullopt;
-    if (positionState) {
-        duration = positionState->duration;
-        rate = positionState->playbackRate;
-    }
-    if (currentPosition)
-        currentTime = *currentPosition;
-    if (RefPtr sessionMetadata = session ? session->metadata() : nullptr) {
-        std::optional<NowPlayingInfoArtwork> artwork;
-        if (sessionMetadata->artworkImage()) {
-            ASSERT(sessionMetadata->artworkImage()->data(), "An image must always have associated data");
-            artwork = NowPlayingInfoArtwork { sessionMetadata->artworkSrc(), sessionMetadata->artworkImage()->mimeType(), sessionMetadata->artworkImage() };
-        }
-        return NowPlayingInfo { sessionMetadata->title(), sessionMetadata->artist(), sessionMetadata->album(), sourceApplicationIdentifier, duration, currentTime, rate, supportsSeeking, m_element.mediaUniqueIdentifier(), isPlaying, allowsNowPlayingControlsVisibility, WTFMove(artwork) };
-    }
+    if (RefPtr session = mediaSession())
+        session->updateNowPlayingInfo(info);
 #endif
 
-    return NowPlayingInfo { m_element.mediaSessionTitle(), emptyString(), emptyString(), sourceApplicationIdentifier, duration, currentTime, rate, supportsSeeking, m_element.mediaUniqueIdentifier(), isPlaying, allowsNowPlayingControlsVisibility, { } };
+    return info;
 }
 
 void MediaElementSession::updateMediaUsageIfChanged()

Source/WebCore/html/MediaElementSession.h

@@ -145,7 +145,6 @@ class MediaElementSession final : public PlatformMediaSession {
     bool wantsToObserveViewportVisibilityForMediaControls() const;
     bool wantsToObserveViewportVisibilityForAutoplay() const;
 
-    enum class PlaybackControlsPurpose { ControlsManager, NowPlaying, MediaSession };
     bool canShowControlsManager(PlaybackControlsPurpose) const;
     bool isLargeEnoughForMainContent(MediaSessionMainContentPurpose) const;
     bool isLongEnoughForMainContent() const final;
@@ -161,7 +160,7 @@ class MediaElementSession final : public PlatformMediaSession {
             || type == MediaType::VideoAudio;
     }
 
-    std::optional<NowPlayingInfo> nowPlayingInfo() const final;
+    std::optional<NowPlayingInfo> computeNowPlayingInfo() const;
 
     WEBCORE_EXPORT void updateMediaUsageIfChanged() final;
     std::optional<MediaUsageInfo> mediaUsageInfo() const { return m_mediaUsageInfo; }
@@ -181,6 +180,8 @@ class MediaElementSession final : public PlatformMediaSession {
 
     MediaSession* mediaSession() const;
 
+    bool hasNowPlayingInfo() const;
+
 private:
 
 #if ENABLE(WIRELESS_PLAYBACK_TARGET)

Source/WebCore/platform/audio/PlatformMediaSession.cpp

@@ -439,7 +439,17 @@ bool PlatformMediaSession::shouldOverridePauseDuringRouteChange() const
 
 std::optional<NowPlayingInfo> PlatformMediaSession::nowPlayingInfo() const
 {
-    return { };
+    return client().nowPlayingInfo();
+}
+
+bool PlatformMediaSession::isNowPlayingEligible() const
+{
+    return client().isNowPlayingEligible();
+};
+
+WeakPtr<PlatformMediaSession> PlatformMediaSession::selectBestMediaSession(const Vector<WeakPtr<PlatformMediaSession>>& sessions, PlaybackControlsPurpose purpose)
+{
+    return client().selectBestMediaSession(sessions, purpose);
 }
 
 #if !RELEASE_LOG_DISABLED
@@ -454,6 +464,11 @@ MediaTime PlatformMediaSessionClient::mediaSessionDuration() const
     return MediaTime::invalidTime();
 }
 
+std::optional<NowPlayingInfo> PlatformMediaSessionClient::nowPlayingInfo() const
+{
+    return { };
+}
+
 }
 
 #endif

Source/WebCore/platform/audio/PlatformMediaSession.h

@@ -126,6 +126,7 @@ class PlatformMediaSession
     void setActive(bool);
 
     using MediaType = WebCore::PlatformMediaSessionMediaType;
+    enum class PlaybackControlsPurpose { ControlsManager, NowPlaying, MediaSession };
 
     MediaType mediaType() const;
     MediaType presentationType() const;
@@ -229,7 +230,10 @@ class PlatformMediaSession
         virtual bool wantsToCaptureAudio() const = 0;
     };
 
-    virtual std::optional<NowPlayingInfo> nowPlayingInfo() const;
+    std::optional<NowPlayingInfo> nowPlayingInfo() const;
+    bool isNowPlayingEligible() const;
+    WeakPtr<PlatformMediaSession> selectBestMediaSession(const Vector<WeakPtr<PlatformMediaSession>>&, PlaybackControlsPurpose);
+
     virtual void updateMediaUsageIfChanged() { }
 
     virtual bool isLongEnoughForMainContent() const { return false; }
@@ -307,6 +311,10 @@ class PlatformMediaSessionClient {
 
     virtual bool shouldOverridePauseDuringRouteChange() const { return false; }
 
+    virtual bool isNowPlayingEligible() const { return false; }
+    virtual std::optional<NowPlayingInfo> nowPlayingInfo() const;
+    virtual WeakPtr<PlatformMediaSession> selectBestMediaSession(const Vector<WeakPtr<PlatformMediaSession>>&, PlatformMediaSession::PlaybackControlsPurpose) { return nullptr; }
+
 #if !RELEASE_LOG_DISABLED
     virtual const Logger& logger() const = 0;
 #endif

Source/WebCore/platform/audio/PlatformMediaSessionManager.cpp

@@ -834,6 +834,26 @@ void PlatformMediaSessionManager::setMediaCapabilityGrantsEnabled(bool mediaCapa
 }
 #endif
 
+WeakPtr<PlatformMediaSession> PlatformMediaSessionManager::bestEligibleSessionForRemoteControls(const Function<bool(const PlatformMediaSession&)>& filterFunction, PlatformMediaSession::PlaybackControlsPurpose purpose)
+{
+    Vector<WeakPtr<PlatformMediaSession>> eligibleAudioVideoSessions;
+    Vector<WeakPtr<PlatformMediaSession>> eligibleWebAudioSessions;
+    forEachMatchingSession(filterFunction, [&](auto& session) {
+        if (eligibleAudioVideoSessions.isEmpty() && session.presentationType() == PlatformMediaSession::MediaType::WebAudio)
+            eligibleWebAudioSessions.append(session);
+        else
+            eligibleAudioVideoSessions.append(session);
+    });
+
+    if (eligibleAudioVideoSessions.isEmpty()) {
+        if (eligibleWebAudioSessions.isEmpty())
+            return nullptr;
+        return eligibleWebAudioSessions[0]->selectBestMediaSession(eligibleWebAudioSessions, purpose);
+    }
+
+    return eligibleAudioVideoSessions[0]->selectBestMediaSession(eligibleAudioVideoSessions, purpose);
+}
+
 #if !RELEASE_LOG_DISABLED
 WTFLogChannel& PlatformMediaSessionManager::logChannel() const
 {

Source/WebCore/platform/audio/PlatformMediaSessionManager.h

@@ -189,6 +189,8 @@ class PlatformMediaSessionManager
 
     bool isApplicationInBackground() const { return m_isApplicationInBackground; }
 
+    WeakPtr<PlatformMediaSession> bestEligibleSessionForRemoteControls(const Function<bool(const PlatformMediaSession&)>&, PlatformMediaSession::PlaybackControlsPurpose);
+
 protected:
     friend class PlatformMediaSession;
     static std::unique_ptr<PlatformMediaSessionManager> create();

Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.h

@@ -97,7 +97,7 @@ class MediaSessionManagerCocoa
 
     virtual void providePresentingApplicationPIDIfNecessary() { }
 
-    PlatformMediaSession* nowPlayingEligibleSession();
+    WeakPtr<PlatformMediaSession> nowPlayingEligibleSession();
 
     void addSupportedCommand(PlatformMediaSession::RemoteControlCommandType) final;
     void removeSupportedCommand(PlatformMediaSession::RemoteControlCommandType) final;

Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.mm

@@ -30,7 +30,6 @@
 
 #import "AudioUtilities.h"
 #import "DeprecatedGlobalSettings.h"
-#import "HTMLMediaElement.h"
 #import "Logging.h"
 #import "MediaConfiguration.h"
 #import "MediaPlayer.h"
@@ -465,13 +464,11 @@
     }
 }
 
-PlatformMediaSession* MediaSessionManagerCocoa::nowPlayingEligibleSession()
+WeakPtr<PlatformMediaSession> MediaSessionManagerCocoa::nowPlayingEligibleSession()
 {
-    // FIXME: Fix this layering violation.
-    if (auto element = HTMLMediaElement::bestMediaElementForRemoteControls(MediaElementSession::PlaybackControlsPurpose::NowPlaying))
-        return &element->mediaSession();
-
-    return nullptr;
+    return bestEligibleSessionForRemoteControls([](auto& session) {
+        return session.isNowPlayingEligible();
+    }, PlatformMediaSession::PlaybackControlsPurpose::NowPlaying);
 }
 
 void MediaSessionManagerCocoa::updateNowPlayingInfo()
@@ -482,7 +479,7 @@
     BEGIN_BLOCK_OBJC_EXCEPTIONS
 
     std::optional<NowPlayingInfo> nowPlayingInfo;
-    if (auto* session = nowPlayingEligibleSession())
+    if (auto session = nowPlayingEligibleSession())
         nowPlayingInfo = session->nowPlayingInfo();
 
     if (!nowPlayingInfo) {