[EME] Blocked encrypted samples are not enqueued after a CDM is attac…

…hed to a SourceBuffer (affects netflix.com)

https://bugs.webkit.org/show_bug.cgi?id=267804
rdar://120879185

Reviewed by Jean-Yves Avenard and Jer Noble.

When SourceBuffer attempts to enqueue an encrypted sample before either a CDM is attached or a
usable key is present it adds the sample to a list of "blocked" samples. The intention was to later
enqueue these samples once a CDM was attached and usable keys were present, however this would
never occur for two reasons:

1. While SourceBufferPrivateAVFObjC had a key status observer that would enqueue blocked samples
when called, it was never added to the attached CDM and therefore never called.
2. Even if the observer were added, if key status resolved to a terminal state prior to the CDM
being attached then the observer would never be called.

These issues manifested in sporadic playback failures on netflix.com (and possibly other EME sites).

Addressed (1) by adding SourceBufferPrivateAVFObjC's key status observer to the attached CDM.
Addressed (2) by attempting to enqueue blocked samples immediately at CDM attachment time.

Added a layout test.

* LayoutTests/http/tests/media/fairplay/eme2016.js:
(async startEME):
(async fetchAppendAndWaitForEncrypted):
(async createBufferAppendAndWaitForEncrypted):
* LayoutTests/http/tests/media/fairplay/fps-mse-attach-cdm-after-key-exchange-expected.txt: Added.
* LayoutTests/http/tests/media/fairplay/fps-mse-attach-cdm-after-key-exchange.html: Copied from LayoutTests/http/tests/media/fairplay/fps-mse-play-while-not-in-dom.html.
* LayoutTests/http/tests/media/fairplay/fps-mse-play-while-not-in-dom.html:
* LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-audio-only.html:
* LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-key-renewal.html:
* LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-key-rotation.html:
* LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-multiple-keys.html:
* LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-same-key.html:
* LayoutTests/platform/mac/TestExpectations:
* Source/WebCore/html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::enterFullscreen):
* Source/WebCore/platform/graphics/avfoundation/objc/CDMInstanceFairPlayStreamingAVFObjC.h:
* Source/WebCore/platform/graphics/avfoundation/objc/CDMInstanceFairPlayStreamingAVFObjC.mm:
(WebCore::CDMInstanceFairPlayStreamingAVFObjC::addKeyStatusesChangedObserver):
(WebCore::CDMInstanceFairPlayStreamingAVFObjC::removeKeyStatusesChangedObserver):
* Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h:
* Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
(WebCore::SourceBufferPrivateAVFObjC::SourceBufferPrivateAVFObjC):
(WebCore::SourceBufferPrivateAVFObjC::setCDMInstance):
(WebCore::SourceBufferPrivateAVFObjC::attemptToDecrypt):
(WebCore::SourceBufferPrivateAVFObjC::tryToEnqueueBlockedSamples):
(WebCore::SourceBufferPrivateAVFObjC::keyStatusesChanged): Renamed to tryToEnqueueBlockedSamples.

Canonical link: https://commits.webkit.org/273340@main

LayoutTests/http/tests/media/fairplay/eme2016.js

@@ -50,8 +50,11 @@ async function startEME(options) {
     let certificate = await fetchBuffer('resources/cert.der');
     await keys.setServerCertificate(certificate);
     consoleWrite('PROMISE: keys.setServerCertificate resolved');
-    await video.setMediaKeys(keys);
-    consoleWrite('PROMISE: setMediaKeys() resolved');
+
+    if (options.setMediaKeys) {
+        await video.setMediaKeys(keys);
+        consoleWrite('PROMISE: setMediaKeys() resolved');
+    }
 
     return keys;
 }
@@ -78,11 +81,11 @@ async function fetchAndWaitForLicenseRequest(session, sourceBuffer, url) {
     });
 }
 
-async function fetchAppendAndWaitForEncrypted(video, sourceBuffer, url) {
+async function fetchAppendAndWaitForEncrypted(video, mediaKeys, sourceBuffer, url) {
     let updateEndPromise = fetchAndAppend(sourceBuffer, url, true);
     let encryptedEvent = await waitFor(video, 'encrypted');
 
-    let session = video.mediaKeys.createSession();
+    let session = mediaKeys.createSession();
     session.generateRequest(encryptedEvent.initDataType, encryptedEvent.initData);
     let message = await waitFor(session, 'message');
     let response = await getResponse(message);
@@ -100,11 +103,11 @@ async function createBufferAndAppend(mediaSource, type, url) {
     return sourceBuffer;
 }
 
-async function createBufferAppendAndWaitForEncrypted(video, mediaSource, type, url) {
+async function createBufferAppendAndWaitForEncrypted(video, mediaSource, mediaKeys, type, url) {
     let sourceBuffer = mediaSource.addSourceBuffer(type);
     consoleWrite('Created sourceBuffer');
 
-    let session = await fetchAppendAndWaitForEncrypted(video, sourceBuffer, url);
+    let session = await fetchAppendAndWaitForEncrypted(video, mediaKeys, sourceBuffer, url);
 
     return {sourceBuffer, session};
 }

LayoutTests/http/tests/media/fairplay/fps-mse-attach-cdm-after-key-exchange-expected.txt

@@ -0,0 +1,25 @@
+PROMISE: requestMediaKeySystemAccess resolved
+PROMISE: createMediaKeys resolved
+FETCH: resources/cert.der OK
+PROMISE: keys.setServerCertificate resolved
+Created mediaSource
+EVENT(sourceopen)
+-
+Appending Encrypted Video Header
+Created sourceBuffer
+FETCH: content/elementary-stream-video-header-keyid-2.m4v OK
+EVENT(encrypted)
+EVENT(message)
+PROMISE: licenseResponse resolved
+PROMISE: session.update() resolved
+EVENT(updateend)
+-
+Appending Encrypted Video Payload
+FETCH: content/elementary-stream-video-payload.m4v OK
+EVENT(updateend)
+RUN(video.play())
+PROMISE: setMediaKeys() resolved
+EVENT(playing)
+PROMISE: playing event dispatched
+END OF TEST
+

LayoutTests/http/tests/media/fairplay/fps-mse-attach-cdm-after-key-exchange.html

@@ -0,0 +1,52 @@
+<!DOCTYPE html><!-- webkit-test-runner [ SampleBufferContentKeySessionSupportEnabled=true ] -->
+<html>
+<head>
+    <title>fps-mse-attach-cdm-after-key-exchange</title>
+    <script src=../../../media-resources/video-test.js></script>
+    <script src=support.js></script>
+    <script src="eme2016.js"></script>
+    <script>
+    window.addEventListener('load', async event => {
+        startTest().then(endTest).catch(failTest);
+    });
+
+    async function startTest() {
+        window.video = document.querySelector('video');
+        let keys = await startEME({video: video, setMediaKeys: false, capabilities: [{
+            initDataTypes: ['sinf'],
+            videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
+            distinctiveIdentifier: 'not-allowed',
+            persistentState: 'not-allowed',
+            sessionTypes: ['temporary'],
+        }]});
+
+        let mediaSource = new MediaSource;
+        video.srcObject = mediaSource;
+        consoleWrite('Created mediaSource');
+        await waitFor(mediaSource, 'sourceopen');
+
+        consoleWrite('-');
+        consoleWrite('Appending Encrypted Video Header');
+
+        let {sourceBuffer: sourceBuffer, session: session} = await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
+
+        consoleWrite('-');
+        consoleWrite('Appending Encrypted Video Payload');
+
+        await fetchAndAppend(sourceBuffer, 'content/elementary-stream-video-payload.m4v');
+
+        run('video.play()');
+        let playingEvent = waitForEventWithTimeout(video, 'playing', 10000, 'Did not play in time');
+
+        await video.setMediaKeys(keys);
+        consoleWrite('PROMISE: setMediaKeys() resolved');
+
+        await playingEvent;
+        consoleWrite('PROMISE: playing event dispatched');
+    }
+    </script>
+</head>
+<body>
+    <video controls width="480"></video>
+</body>
+</html>

LayoutTests/http/tests/media/fairplay/fps-mse-play-while-not-in-dom.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         window.video = document.createElement('video');
-        let keys = await startEME({video: video, capabilities: [{
+        let keys = await startEME({video: video, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
             distinctiveIdentifier: 'not-allowed',
@@ -28,7 +28,7 @@
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Header');
 
-        let {sourceBuffer: sourceBuffer, session: session} = await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
+        let {sourceBuffer: sourceBuffer, session: session} = await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Payload');

LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-audio-only.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         window.audio = document.querySelector('audio');
-        let keys = await startEME({video: audio, capabilities: [{
+        let keys = await startEME({video: audio, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             audioCapabilities: [{ contentType: 'audio/mp4', robustness: '' }],
             distinctiveIdentifier: 'not-allowed',
@@ -28,7 +28,7 @@
         consoleWrite('-');
         consoleWrite('Appending Encrypted Audio Header');
 
-        let {sourceBuffer: sourceBuffer, session: session} = await createBufferAppendAndWaitForEncrypted(audio, mediaSource, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
+        let {sourceBuffer: sourceBuffer, session: session} = await createBufferAppendAndWaitForEncrypted(audio, mediaSource, keys, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Audio Payload');

LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-key-renewal.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         let video = document.querySelector('video');
-        let keys = await startEME({video: video, capabilities: [{
+        let keys = await startEME({video: video, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             audioCapabilities: [{ contentType: 'audio/mp4', robustness: '' }],
             videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
@@ -30,7 +30,7 @@
         consoleWrite('Appending Encrypted Video Buffer 1');
 
         let {sourceBuffer: sourceBuffer1, session: session1} =
-            await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
+            await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Payload');

LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-key-rotation.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         let video = document.querySelector('video');
-        let keys = await startEME({video: video, capabilities: [{
+        let keys = await startEME({video: video, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             audioCapabilities: [{ contentType: 'audio/mp4', robustness: '' }],
             videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
@@ -30,7 +30,7 @@
         consoleWrite('Appending Encrypted Video Buffer 1');
 
         let {sourceBuffer: sourceBuffer1, session: session1} =
-            await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
+            await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Payload');
@@ -40,7 +40,7 @@
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Buffer 2');
 
-        session2 = await fetchAppendAndWaitForEncrypted(video, sourceBuffer1, 'content/elementary-stream-video-header-keyid-4.m4v');
+        session2 = await fetchAppendAndWaitForEncrypted(video, keys, sourceBuffer1, 'content/elementary-stream-video-header-keyid-4.m4v');
 
         consoleWrite('Setting timestampOffset = buffered.end()');
         sourceBuffer1.timestampOffset = sourceBuffer1.buffered.end(0);

LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-multiple-keys.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         let video = document.querySelector('video');
-        let keys = await startEME({video: video, capabilities: [{
+        let keys = await startEME({video: video, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             audioCapabilities: [{ contentType: 'audio/mp4', robustness: '' }],
             videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
@@ -30,13 +30,13 @@
         consoleWrite('Appending Encrypted Audio Header');
 
         let {sourceBuffer: sourceBuffer1, session: session1} =
-            await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
+            await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Header');
 
         let {sourceBuffer: sourceBuffer2, session: session2} =
-            await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
+            await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'video/mp4', 'content/elementary-stream-video-header-keyid-2.m4v');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Audio Payload');

LayoutTests/http/tests/media/fairplay/fps-mse-unmuxed-same-key.html

@@ -12,7 +12,7 @@
 
     async function startTest() {
         let video = document.querySelector('video');
-        let keys = await startEME({video: video, capabilities: [{
+        let keys = await startEME({video: video, setMediaKeys: true, capabilities: [{
             initDataTypes: ['sinf'],
             audioCapabilities: [{ contentType: 'audio/mp4', robustness: '' }],
             videoCapabilities: [{ contentType: 'video/mp4', robustness: '' }],
@@ -29,7 +29,7 @@
         consoleWrite('-');
         consoleWrite('Appending Encrypted Audio Header');
 
-        let {sourceBuffer: sourceBuffer1, session: session1} = await createBufferAppendAndWaitForEncrypted(video, mediaSource, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
+        let {sourceBuffer: sourceBuffer1, session: session1} = await createBufferAppendAndWaitForEncrypted(video, mediaSource, keys, 'audio/mp4', 'content/elementary-stream-audio-header-keyid-1.m4a');
 
         consoleWrite('-');
         consoleWrite('Appending Encrypted Video Header');

LayoutTests/platform/mac/TestExpectations

@@ -95,6 +95,8 @@ imported/w3c/web-platform-tests/pointerevents [ Pass ]
 imported/w3c/web-platform-tests/speech-api [ Pass ]
 
 http/tests/media/fairplay [ Pass ]
+http/tests/media/fairplay/fps-mse-attach-cdm-after-key-exchange.html [ Skip ] # Requires SampleBufferContentKeySessionSupportEnabled
+
 media/track/track-description-cue.html [ Pass ]
 media/track/track-extended-descriptions.html [ Pass ]
 media/media-source/remoteplayback-from-source-element.html [ Pass ]

Source/WebCore/html/HTMLMediaElement.cpp

@@ -6820,7 +6820,7 @@ void HTMLMediaElement::enterFullscreen(VideoFullscreenMode mode)
 
 #if ENABLE(FULLSCREEN_API) && ENABLE(VIDEO_USES_ELEMENT_FULLSCREEN)
 #if PLATFORM(IOS_FAMILY)
-    bool videoUsesElementFullscreen = document().settings().videoFullscreenRequiresElementFullscreen();
+    bool videoUsesElementFullscreen = document().settings().videoFullscreenRequiresElementFullscreen() && !document().settings().newThing();
 #else
     constexpr bool videoUsesElementFullscreen = true;
 #endif

Source/WebCore/platform/graphics/avfoundation/objc/CDMInstanceFairPlayStreamingAVFObjC.h

@@ -137,6 +137,7 @@ class CDMInstanceFairPlayStreamingAVFObjC final : public CDMInstance, public AVC
 
     using KeyStatusesChangedObserver = Observer<void()>;
     void addKeyStatusesChangedObserver(const KeyStatusesChangedObserver&);
+    void removeKeyStatusesChangedObserver(const KeyStatusesChangedObserver&);
 
     void sessionKeyStatusesChanged(const CDMInstanceSessionFairPlayStreamingAVFObjC&);
 

Source/WebCore/platform/graphics/avfoundation/objc/CDMInstanceFairPlayStreamingAVFObjC.mm

@@ -706,9 +706,16 @@ static bool isPotentiallyUsableKeyStatus(CDMInstanceSession::KeyStatus status)
 
 void CDMInstanceFairPlayStreamingAVFObjC::addKeyStatusesChangedObserver(const KeyStatusesChangedObserver& observer)
 {
+    ASSERT(!m_keyStatusChangedObservers.contains(observer));
     m_keyStatusChangedObservers.add(observer);
 }
 
+void CDMInstanceFairPlayStreamingAVFObjC::removeKeyStatusesChangedObserver(const KeyStatusesChangedObserver& observer)
+{
+    ASSERT(m_keyStatusChangedObservers.contains(observer));
+    m_keyStatusChangedObservers.remove(observer);
+}
+
 void CDMInstanceFairPlayStreamingAVFObjC::sessionKeyStatusesChanged(const CDMInstanceSessionFairPlayStreamingAVFObjC&)
 {
     m_keyStatusChangedObservers.forEach([] (auto& observer) {

Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h

@@ -198,7 +198,7 @@ ALLOW_NEW_API_WITHOUT_GUARDS_END
     bool trackIsBlocked(TrackID) const;
 
 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
-    void keyStatusesChanged();
+    void tryToEnqueueBlockedSamples();
 #endif
 
     void setTrackChangeCallbacks(const InitializationSegment&, bool initialized);

Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm

@@ -121,7 +121,7 @@ static inline bool shouldAddContentKeyRecipients()
     , m_listener(WebAVSampleBufferListener::create(*this))
     , m_appendQueue(WorkQueue::create("SourceBufferPrivateAVFObjC data parser queue"))
 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
-    , m_keyStatusesChangedObserver(makeUniqueRef<Observer<void()>>([this] { keyStatusesChanged(); }))
+    , m_keyStatusesChangedObserver(makeUniqueRef<Observer<void()>>([this] { tryToEnqueueBlockedSamples(); }))
     , m_streamDataParser(is<SourceBufferParserAVFObjC>(m_parser) ? downcast<SourceBufferParserAVFObjC>(m_parser)->streamDataParser() : nil)
 #endif
 #if !RELEASE_LOG_DISABLED
@@ -669,22 +669,28 @@ static inline bool shouldAddContentKeyRecipients()
 
     ALWAYS_LOG(LOGIDENTIFIER);
 
-    if (m_cdmInstance && shouldAddContentKeyRecipients()) {
-        if (m_videoLayer)
-            [m_cdmInstance->contentKeySession() removeContentKeyRecipient:m_videoLayer->displayLayer()];
+    if (m_cdmInstance) {
+        if (shouldAddContentKeyRecipients()) {
+            if (m_videoLayer)
+                [m_cdmInstance->contentKeySession() removeContentKeyRecipient:m_videoLayer->displayLayer()];
 
-        for (auto& pair : m_audioRenderers)
-            [m_cdmInstance->contentKeySession() removeContentKeyRecipient:pair.second.get()];
+            for (auto& pair : m_audioRenderers)
+                [m_cdmInstance->contentKeySession() removeContentKeyRecipient:pair.second.get()];
+        }
+        m_cdmInstance->removeKeyStatusesChangedObserver(*m_keyStatusesChangedObserver);
     }
 
     m_cdmInstance = fpsInstance;
 
-    if (m_cdmInstance && shouldAddContentKeyRecipients()) {
-        if (m_videoLayer)
-            [m_cdmInstance->contentKeySession() addContentKeyRecipient:m_videoLayer->displayLayer()];
+    if (m_cdmInstance) {
+        if (shouldAddContentKeyRecipients()) {
+            if (m_videoLayer)
+                [m_cdmInstance->contentKeySession() addContentKeyRecipient:m_videoLayer->displayLayer()];
 
-        for (auto& pair : m_audioRenderers)
-            [m_cdmInstance->contentKeySession() addContentKeyRecipient:pair.second.get()];
+            for (auto& pair : m_audioRenderers)
+                [m_cdmInstance->contentKeySession() addContentKeyRecipient:pair.second.get()];
+        }
+        m_cdmInstance->addKeyStatusesChangedObserver(*m_keyStatusesChangedObserver);
     }
 
     attemptToDecrypt();
@@ -712,6 +718,8 @@ static inline bool shouldAddContentKeyRecipients()
         m_hasSessionSemaphore = nullptr;
     }
     m_waitingForKey = false;
+
+    tryToEnqueueBlockedSamples();
 #endif
 }
 #endif // (ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)) || ENABLE(LEGACY_ENCRYPTED_MEDIA)
@@ -929,7 +937,7 @@ static inline bool shouldAddContentKeyRecipients()
 }
 
 #if ENABLE(ENCRYPTED_MEDIA) && HAVE(AVCONTENTKEYSESSION)
-void SourceBufferPrivateAVFObjC::keyStatusesChanged()
+void SourceBufferPrivateAVFObjC::tryToEnqueueBlockedSamples()
 {
     while (!m_blockedSamples.isEmpty()) {
         auto& firstPair = m_blockedSamples.first();